You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by do...@apache.org on 2018/04/03 00:34:34 UTC
[6/9] reef git commit: [REEF-1965] Implement REEF runtime for Azure
Batch
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java
new file mode 100644
index 0000000..b4dc7e4
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureBatchTaskStatusCheckPeriod.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The period to check for task status from Azure Batch in ms.
+ */
+@NamedParameter(doc = "The period to check for task status from Azure Batch in ms.", default_value = "5000")
+public final class AzureBatchTaskStatusCheckPeriod implements Name<Integer> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java
new file mode 100644
index 0000000..f82cf50
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountKey.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The Azure Storage account key.
+ */
+@NamedParameter(doc = "The Azure Storage account key.")
+public final class AzureStorageAccountKey implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java
new file mode 100644
index 0000000..8e0af3e
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageAccountName.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The Azure Storage account name.
+ */
+@NamedParameter(doc = "The Azure Storage account name.")
+public final class AzureStorageAccountName implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java
new file mode 100644
index 0000000..4bd8644
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageBlobSASTokenValidityHours.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The Azure Blob Shared Access Signature token validity in hours.
+ */
+@NamedParameter(doc = "The Azure Blob Shared Access Signature token validity in hours.", default_values = "1")
+public final class AzureStorageBlobSASTokenValidityHours implements Name<Integer> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java
new file mode 100644
index 0000000..ef32d9e
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/AzureStorageContainerName.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The Azure Storage container name.
+ */
+@NamedParameter(doc = "The Azure Storage container name.")
+public final class AzureStorageContainerName implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java
new file mode 100644
index 0000000..00139db
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/ContainerIdentifier.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The container identifier for the evaluator shim.
+ */
+@NamedParameter(doc = "The container identifier.")
+public final class ContainerIdentifier implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java
new file mode 100644
index 0000000..70ca902
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/EvaluatorShimConfigFilePath.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * Evaluator shim config file path parameter.
+ */
+@NamedParameter(doc = "The path to the configuration file for the evaluator shim.")
+public final class EvaluatorShimConfigFilePath implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java
new file mode 100644
index 0000000..8b26db5
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/IsWindows.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * Whether or not the Azure Batch VMs are Windows based.
+ */
+@NamedParameter(doc = "Whether or not the Azure Batch VMs are Windows based.")
+public final class IsWindows implements Name<Boolean> {
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java
new file mode 100644
index 0000000..e00140d
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/parameters/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Client for the REEF runtime for Azure Batch.
+ */
+package org.apache.reef.runtime.azbatch.parameters;
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java
new file mode 100644
index 0000000..1eb4e05
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/AzureBatchFileNames.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util;
+
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.common.files.REEFFileNames;
+
+import javax.inject.Inject;
+
+/**
+ * Access to the various places things go according to the REEF Azure Batch runtime.
+ */
+@Private
+public final class AzureBatchFileNames {
+
+ private static final String STORAGE_JOB_FOLDER_PATH = "apps/reef/jobs/";
+ private static final String TASK_JAR_FILE_NAME = "local.jar";
+ private static final String EVALUATOR_RESOURCE_FILES_JAR_NAME = "resources.jar";
+
+ private static final String EVALUATOR_SHIM_CONFIGURATION_NAME = "shim.conf";
+ private static final String TEXTFILE_EXTENSION = ".txt";
+
+ private final REEFFileNames reefFileNames;
+
+ @Inject
+ private AzureBatchFileNames(final REEFFileNames reefFileNames) {
+ this.reefFileNames = reefFileNames;
+ }
+
+ /**
+ * @return The relative path to the folder storing the job assets.
+ */
+ public String getStorageJobFolder(final String jobId) {
+ return STORAGE_JOB_FOLDER_PATH + jobId;
+ }
+
+ /**
+ * @return The name used within the current working directory of the driver to redirect standard error to.
+ */
+ public String getEvaluatorStdErrFilename() {
+ return this.reefFileNames.getEvaluatorStderrFileName() + TEXTFILE_EXTENSION;
+ }
+
+ /**
+ * @return The name used within the current working directory of the driver to redirect standard out to.
+ */
+ public String getEvaluatorStdOutFilename() {
+ return this.reefFileNames.getEvaluatorStdoutFileName() + TEXTFILE_EXTENSION;
+ }
+
+ /**
+ * @return The path to the evaluator shim configuration:
+ * REEF_BASE_FOLDER/LOCAL_FOLDER/EVALUATOR_SHIM_CONFIGURATION_NAME.
+ */
+ public String getEvaluatorShimConfigurationPath() {
+ return this.reefFileNames.getLocalFolderPath() + "/" + EVALUATOR_SHIM_CONFIGURATION_NAME;
+ }
+
+ /**
+ * @return The name of the evaluator resource JAR file.
+ */
+ public String getEvaluatorResourceFilesJarName() {
+ return EVALUATOR_RESOURCE_FILES_JAR_NAME;
+ }
+
+ /**
+ * @return The name of the evaluator configuration file.
+ */
+ public String getEvaluatorShimConfigurationName() {
+ return EVALUATOR_SHIM_CONFIGURATION_NAME;
+ }
+
+ /**
+ * @return The name under which the task jar will be stored.
+ */
+ public static String getTaskJarFileName() {
+ return TASK_JAR_FILE_NAME;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java
new file mode 100644
index 0000000..3275648
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/RemoteIdentifierParser.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util;
+
+/**
+ * Utility class to parse IP and port information from Azure Batch Node ID.
+ */
+public final class RemoteIdentifierParser {
+
+ private static final String PROTOCOL = "socket://";
+
+ private RemoteIdentifierParser() {}
+
+ /**
+ * Get the IP address from the remote identifier.
+ *
+ * @param remoteIdentifier the remote identifier.
+ * @return the IP address.
+ */
+ public static String parseIp(final String remoteIdentifier) {
+ return remoteIdentifier.substring(PROTOCOL.length(), remoteIdentifier.lastIndexOf(':'));
+ }
+
+ /**
+ * Get the port from the remote identifier.
+ *
+ * @param remoteIdentifier the remote identifier.
+ * @return the port.
+ */
+ public static int parsePort(final String remoteIdentifier) {
+ return Integer.parseInt(remoteIdentifier.substring(remoteIdentifier.lastIndexOf(':') + 1));
+ }
+
+ /**
+ * Get the node ID from the remote identifier.
+ *
+ * @param remoteIdentifier the remote identifier.
+ * @return the node ID.
+ */
+ public static String parseNodeId(final String remoteIdentifier) {
+ return remoteIdentifier.substring(PROTOCOL.length());
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java
new file mode 100644
index 0000000..69d44e9
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/TaskStatusMapper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util;
+
+import com.microsoft.azure.batch.protocol.models.CloudTask;
+import com.microsoft.azure.batch.protocol.models.TaskExecutionResult;
+import org.apache.reef.runtime.common.driver.evaluator.pojos.State;
+
+/**
+ * Class that maps status of Azure Batch task to a REEF task.
+ */
+public final class TaskStatusMapper {
+
+ private TaskStatusMapper() {
+ }
+
+ /**
+ * Get the {@link State} from a {@link CloudTask}.
+ *
+ * @param task the task.
+ * @return the state of the task.
+ */
+ public static State getReefTaskState(final CloudTask task) {
+ switch (task.state()) {
+ case ACTIVE:
+ return State.INIT;
+ case RUNNING:
+ return State.RUNNING;
+ case COMPLETED:
+ if (task.executionInfo().result() == TaskExecutionResult.SUCCESS) {
+ return State.DONE;
+ } else {
+ return State.FAILED;
+ }
+ case PREPARING:
+ return State.RUNNING;
+ default:
+ throw new IllegalArgumentException("Azure batch cloud task has unknown state: " + task.state());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java
new file mode 100644
index 0000000..e0753df
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/AzureBatchHelper.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.batch;
+
+import com.microsoft.azure.batch.BatchClient;
+import com.microsoft.azure.batch.protocol.models.*;
+
+import org.apache.reef.runtime.azbatch.client.AzureBatchJobSubmissionHandler;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchPoolId;
+import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames;
+import org.apache.reef.runtime.azbatch.util.storage.SharedAccessSignatureCloudBlobClientProvider;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A helper class for Azure Batch.
+ */
+public final class AzureBatchHelper {
+
+ private static final Logger LOG = Logger.getLogger(AzureBatchJobSubmissionHandler.class.getName());
+
+ /*
+ * Environment variable that contains the Azure Batch jobId.
+ */
+ private static final String AZ_BATCH_JOB_ID_ENV = "AZ_BATCH_JOB_ID";
+
+ private final AzureBatchFileNames azureBatchFileNames;
+
+ private final BatchClient client;
+ private final PoolInformation poolInfo;
+
+ @Inject
+ public AzureBatchHelper(
+ final AzureBatchFileNames azureBatchFileNames,
+ final IAzureBatchCredentialProvider credentialProvider,
+ @Parameter(AzureBatchPoolId.class) final String azureBatchPoolId) {
+ this.azureBatchFileNames = azureBatchFileNames;
+
+ this.client = BatchClient.open(credentialProvider.getCredentials());
+ this.poolInfo = new PoolInformation().withPoolId(azureBatchPoolId);
+ }
+
+ /**
+ * Create a job on Azure Batch.
+ *
+ * @param applicationId the ID of the application.
+ * @param storageContainerSAS the publicly accessible uri to the job container.
+ * @param jobJarUri the publicly accessible uri to the job jar directory.
+ * @param command the commandline argument to execute the job.
+ * @throws IOException
+ */
+ public void submitJob(final String applicationId, final String storageContainerSAS, final URI jobJarUri,
+ final String command) throws IOException {
+ ResourceFile jarResourceFile = new ResourceFile()
+ .withBlobSource(jobJarUri.toString())
+ .withFilePath(AzureBatchFileNames.getTaskJarFileName());
+
+ // This setting will signal Batch to generate an access token and pass it to the Job Manager Task (aka the Driver)
+ // as an environment variable.
+ // See https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings
+ // for more info.
+ AuthenticationTokenSettings authenticationTokenSettings = new AuthenticationTokenSettings();
+ authenticationTokenSettings.withAccess(Collections.singletonList(AccessScope.JOB));
+
+ EnvironmentSetting environmentSetting = new EnvironmentSetting()
+ .withName(SharedAccessSignatureCloudBlobClientProvider.AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV)
+ .withValue(storageContainerSAS);
+
+ JobManagerTask jobManagerTask = new JobManagerTask()
+ .withRunExclusive(false)
+ .withId(applicationId)
+ .withResourceFiles(Collections.singletonList(jarResourceFile))
+ .withEnvironmentSettings(Collections.singletonList(environmentSetting))
+ .withAuthenticationTokenSettings(authenticationTokenSettings)
+ .withCommandLine(command);
+
+ LOG.log(Level.INFO, "Job Manager (aka driver) task command: " + command);
+
+ JobAddParameter jobAddParameter = new JobAddParameter()
+ .withId(applicationId)
+ .withJobManagerTask(jobManagerTask)
+ .withPoolInfo(poolInfo);
+
+ client.jobOperations().createJob(jobAddParameter);
+ }
+
+ /**
+ * Adds a single task to a job on Azure Batch.
+ *
+ * @param jobId the ID of the job.
+ * @param taskId the ID of the task.
+ * @param jobJarUri the publicly accessible uri list to the job jar directory.
+ * @param confUri the publicly accessible uri list to the job configuration directory.
+ * @param command the commandline argument to execute the job.
+ * @throws IOException
+ */
+ public void submitTask(final String jobId, final String taskId, final URI jobJarUri,
+ final URI confUri, final String command)
+ throws IOException {
+
+ final List<ResourceFile> resources = new ArrayList<>();
+
+ final ResourceFile jarSourceFile = new ResourceFile()
+ .withBlobSource(jobJarUri.toString())
+ .withFilePath(AzureBatchFileNames.getTaskJarFileName());
+ resources.add(jarSourceFile);
+
+ final ResourceFile confSourceFile = new ResourceFile()
+ .withBlobSource(confUri.toString())
+ .withFilePath(this.azureBatchFileNames.getEvaluatorShimConfigurationPath());
+ resources.add(confSourceFile);
+
+ LOG.log(Level.INFO, "Evaluator task command: " + command);
+
+ final TaskAddParameter taskAddParameter = new TaskAddParameter()
+ .withId(taskId)
+ .withResourceFiles(resources)
+ .withCommandLine(command);
+
+ this.client.taskOperations().createTask(jobId, taskAddParameter);
+ }
+
+ /**
+ * List the tasks of the specified job.
+ *
+ * @param jobId the ID of the job.
+ * @return A list of CloudTask objects.
+ */
+ public List<CloudTask> getTaskStatusForJob(final String jobId) {
+ List<CloudTask> tasks = null;
+ try {
+ tasks = client.taskOperations().listTasks(jobId);
+ LOG.log(Level.INFO, "Task status for job: {0} returned {1} tasks", new Object[]{jobId, tasks.size()});
+ } catch (IOException | BatchErrorException ex) {
+ LOG.log(Level.SEVERE, "Exception when fetching Task status for job: {0}. Exception [{1}]:[2]",
+ new Object[]{jobId, ex.getMessage(), ex.getStackTrace()});
+ }
+
+ return tasks;
+ }
+
+ /**
+ * @return the job ID specified in the current system environment.
+ */
+ public String getAzureBatchJobId() {
+ return System.getenv(AZ_BATCH_JOB_ID_ENV);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java
new file mode 100644
index 0000000..681337d
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/IAzureBatchCredentialProvider.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.batch;
+
+import com.microsoft.azure.batch.auth.BatchCredentials;
+
+/**
+ * An interface for Azure Batch credential providers.
+ */
+public interface IAzureBatchCredentialProvider {
+
+ /**
+ * Returns {@link BatchCredentials} for Azure Batch account.
+ * @return an implementation of {@link BatchCredentials}.
+ */
+ BatchCredentials getCredentials();
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java
new file mode 100644
index 0000000..3e40319
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/SharedKeyBatchCredentialProvider.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.batch;
+
+import com.microsoft.azure.batch.auth.BatchCredentials;
+import com.microsoft.azure.batch.auth.BatchSharedKeyCredentials;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountKey;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountName;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+
+/**
+ * An implementation of {@link IAzureBatchCredentialProvider} which returns {@link BatchSharedKeyCredentials}
+ * for Azure Batch account.
+ */
+@Private
+public final class SharedKeyBatchCredentialProvider implements IAzureBatchCredentialProvider {
+
+ private final String azureBatchAccountUri;
+ private final String azureBatchAccountName;
+ private final String azureBatchAccountKey;
+
+ @Inject
+ private SharedKeyBatchCredentialProvider(@Parameter(AzureBatchAccountUri.class) final String azureBatchAccountUri,
+ @Parameter(AzureBatchAccountName.class) final String azureBatchAccountName,
+ @Parameter(AzureBatchAccountKey.class) final String azureBatchAccountKey) {
+ this.azureBatchAccountUri = azureBatchAccountUri;
+ this.azureBatchAccountName = azureBatchAccountName;
+ this.azureBatchAccountKey = azureBatchAccountKey;
+ }
+
+ /**
+ * Returns credentials for Azure Batch account.
+ * @return an instance of {@link BatchSharedKeyCredentials} based on {@link AzureBatchAccountKey} parameter.
+ */
+ @Override
+ public BatchCredentials getCredentials() {
+ return new BatchSharedKeyCredentials(this.azureBatchAccountUri, this.azureBatchAccountName, azureBatchAccountKey);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java
new file mode 100644
index 0000000..a3b8caa
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/TokenBatchCredentialProvider.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.batch;
+
+import com.microsoft.azure.batch.auth.BatchCredentials;
+import com.microsoft.rest.credentials.TokenCredentials;
+import okhttp3.OkHttpClient;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+
+/**
+ * An implementation of {@link IAzureBatchCredentialProvider} which returns credentials based on
+ * the authentication token provided in AZ_BATCH_AUTHENTICATION_TOKEN environment variable. This environment variable
+ * is populated by Azure Batch and allows the task running in Azure Batch perform certain operations on the job.
+ *
+ * See: https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings
+ * for more info.
+ */
+public class TokenBatchCredentialProvider implements IAzureBatchCredentialProvider {
+
+ private static final String AZ_BATCH_AUTH_TOKEN_ENV = "AZ_BATCH_AUTHENTICATION_TOKEN";
+
+ private final String azureBatchAccountUri;
+
+ @Inject
+ TokenBatchCredentialProvider(@Parameter(AzureBatchAccountUri.class) final String azureBatchAccountUri) {
+ this.azureBatchAccountUri = azureBatchAccountUri;
+ }
+
+ /**
+ * Returns credentials for Azure Batch account.
+ * @return an implementation of {@link BatchCredentials} which is based on the token provided by Azure Batch.
+ */
+ @Override
+ public BatchCredentials getCredentials() {
+
+ final TokenCredentials tokenCredentials = new TokenCredentials(null, System.getenv(AZ_BATCH_AUTH_TOKEN_ENV));
+
+ return new BatchCredentials() {
+ @Override
+ public String baseUrl() {
+ return azureBatchAccountUri;
+ }
+
+ @Override
+ public void applyCredentialsFilter(final OkHttpClient.Builder builder) {
+ tokenCredentials.applyCredentialsFilter(builder);
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java
new file mode 100644
index 0000000..830749e
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/batch/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Azure Batch utilities for REEF Azure Batch runtime.
+ */
+package org.apache.reef.runtime.azbatch.util.batch;
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java
new file mode 100644
index 0000000..9a6cc15
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/AbstractCommandBuilder.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.command;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.reef.driver.evaluator.EvaluatorProcess;
+import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames;
+import org.apache.reef.runtime.common.client.api.JobSubmissionEvent;
+import org.apache.reef.runtime.common.driver.api.ResourceLaunchEvent;
+import org.apache.reef.runtime.common.files.ClasspathProvider;
+import org.apache.reef.runtime.common.files.REEFFileNames;
+import org.apache.reef.runtime.common.files.RuntimePathProvider;
+import org.apache.reef.runtime.common.launch.JavaLaunchCommandBuilder;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Abstract implementation of the OS command builder.
+ */
+public abstract class AbstractCommandBuilder implements CommandBuilder {
+
+ private final Class launcherClass;
+ private final Class shimLauncherClass;
+ private final List<String> commandListPrefix;
+ private final String osCommandFormat;
+ private final RuntimePathProvider runtimePathProvider;
+
+ protected final ClasspathProvider classpathProvider;
+ protected final REEFFileNames reefFileNames;
+ protected final AzureBatchFileNames azureBatchFileNames;
+
+ AbstractCommandBuilder(
+ final Class launcherClass,
+ final Class shimLauncherClass,
+ final List<String> commandListPrefix,
+ final String osCommandFormat,
+ final ClasspathProvider classpathProvider,
+ final RuntimePathProvider runtimePathProvider,
+ final REEFFileNames reefFileNames,
+ final AzureBatchFileNames azureBatchFileNames) {
+ this.launcherClass = launcherClass;
+ this.shimLauncherClass = shimLauncherClass;
+ this.commandListPrefix = commandListPrefix;
+ this.osCommandFormat = osCommandFormat;
+
+ this.classpathProvider = classpathProvider;
+ this.reefFileNames = reefFileNames;
+ this.azureBatchFileNames = azureBatchFileNames;
+ this.runtimePathProvider = runtimePathProvider;
+ }
+
+ public String buildDriverCommand(final JobSubmissionEvent jobSubmissionEvent) {
+ List<String> commandList = new JavaLaunchCommandBuilder(this.launcherClass, this.commandListPrefix)
+ .setJavaPath(runtimePathProvider.getPath())
+ .setConfigurationFilePaths(Collections.singletonList(this.reefFileNames.getDriverConfigurationPath()))
+ .setClassPath(getDriverClasspath())
+ .setMemory(jobSubmissionEvent.getDriverMemory().get())
+ .build();
+ return String.format(this.osCommandFormat, StringUtils.join(commandList, ' '));
+ }
+
+ public String buildEvaluatorShimCommand(final int evaluatorShimMemory, final String configurationPath) {
+ List<String> commandList = new JavaLaunchCommandBuilder(this.shimLauncherClass, this.commandListPrefix)
+ .setJavaPath(runtimePathProvider.getPath())
+ .setConfigurationFilePaths(Collections.singletonList(configurationPath))
+ .setClassPath(getEvaluatorShimClasspath())
+ .setMemory(evaluatorShimMemory)
+ .build();
+ return String.format(this.osCommandFormat, StringUtils.join(commandList, ' '));
+ }
+
+ public String buildEvaluatorCommand(final ResourceLaunchEvent resourceLaunchEvent,
+ final int containerMemory, final double jvmHeapFactor) {
+ List<String> commandList = new ArrayList<>();
+
+ // Use EvaluatorProcess to be compatible with JVMProcess and CLRProcess
+ final EvaluatorProcess process = resourceLaunchEvent.getProcess()
+ .setConfigurationFileName(this.reefFileNames.getEvaluatorConfigurationPath());
+
+ if (process.isOptionSet()) {
+ commandList.addAll(process.getCommandLine());
+ } else {
+ commandList.addAll(process.setMemory((int) (jvmHeapFactor * containerMemory)).getCommandLine());
+ }
+
+ return StringUtils.join(commandList, ' ');
+ }
+
+ /**
+ * Returns the driver classpath string which is compatible with the intricacies of the OS.
+ *
+ * @return classpath parameter string.
+ */
+ protected abstract String getDriverClasspath();
+
+ /**
+ * Returns the evaluator shim classpath string which is compatible with the intricacies of the OS.
+ *
+ * @return classpath parameter string.
+ */
+ protected abstract String getEvaluatorShimClasspath();
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java
new file mode 100644
index 0000000..96cd716
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/CommandBuilder.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.command;
+
+import org.apache.reef.runtime.common.client.api.JobSubmissionEvent;
+import org.apache.reef.runtime.common.driver.api.ResourceLaunchEvent;
+
+/**
+ * Build the launch command for Java REEF processes for Azure Batch.
+ */
+public interface CommandBuilder {
+
+ /**
+ * Assembles the command to execute the Driver.
+ *
+ * @param jobSubmissionEvent the submission event for which to build the command.
+ * @return the command string.
+ */
+ String buildDriverCommand(JobSubmissionEvent jobSubmissionEvent);
+
+ /**
+ * Assembles the command to execute the Evaluator Shim.
+ *
+ * @param evaluatorShimMemory the maximum size in, in bytes, of the memory allocation pool.
+ * @param configurationPath the relative path to the configuration file.
+ * @return the command string.
+ */
+ String buildEvaluatorShimCommand(int evaluatorShimMemory, String configurationPath);
+
+ /**
+ * Assembles the command to execute the Evaluator.
+ *
+ * @param resourceLaunchEvent the launch event for which to build the command.
+ * @param containerMemory the maximum size in, in bytes, of the memory allocation pool.
+ * @param jvmHeapFactor a multiplicative factor to increase the container memory.
+ * @return the command string.
+ */
+ String buildEvaluatorCommand(final ResourceLaunchEvent resourceLaunchEvent,
+ final int containerMemory, final double jvmHeapFactor);
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java
new file mode 100644
index 0000000..adb58f5
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/LinuxCommandBuilder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.command;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher;
+import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames;
+import org.apache.reef.runtime.common.REEFLauncher;
+import org.apache.reef.runtime.common.files.ClasspathProvider;
+import org.apache.reef.runtime.common.files.REEFFileNames;
+import org.apache.reef.runtime.common.files.RuntimePathProvider;
+
+import javax.inject.Inject;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Build the launch command for Java REEF processes for Azure Batch Linux pools.
+ */
+@Private
+public class LinuxCommandBuilder extends AbstractCommandBuilder {
+
+ private static final Class LAUNCHER_CLASS = REEFLauncher.class;
+ private static final Class SHIM_LAUNCHER_CLASS = EvaluatorShimLauncher.class;
+ private static final List<String> COMMAND_LIST_PREFIX =
+ Collections.unmodifiableList(Arrays.asList(
+ "unzip " + AzureBatchFileNames.getTaskJarFileName() + " -d 'reef/'" + ";"));
+ private static final char CLASSPATH_SEPARATOR_CHAR = ':';
+ private static final String OS_COMMAND_FORMAT = "/bin/sh -c \"%s\"";
+
+ @Inject
+ LinuxCommandBuilder(
+ final ClasspathProvider classpathProvider,
+ final RuntimePathProvider runtimePathProvider,
+ final REEFFileNames reefFileNames,
+ final AzureBatchFileNames azureBatchFileNames) {
+ super(LAUNCHER_CLASS, SHIM_LAUNCHER_CLASS, COMMAND_LIST_PREFIX, OS_COMMAND_FORMAT,
+ classpathProvider, runtimePathProvider, reefFileNames, azureBatchFileNames);
+ }
+
+ @Override
+ protected String getDriverClasspath() {
+ return StringUtils.join(super.classpathProvider.getDriverClasspath(), CLASSPATH_SEPARATOR_CHAR);
+ }
+
+ @Override
+ protected String getEvaluatorShimClasspath() {
+ return StringUtils.join(super.classpathProvider.getEvaluatorClasspath(), CLASSPATH_SEPARATOR_CHAR);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java
new file mode 100644
index 0000000..f382bef
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/WindowsCommandBuilder.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.command;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher;
+import org.apache.reef.runtime.azbatch.util.AzureBatchFileNames;
+import org.apache.reef.runtime.common.REEFLauncher;
+import org.apache.reef.runtime.common.files.ClasspathProvider;
+import org.apache.reef.runtime.common.files.REEFFileNames;
+import org.apache.reef.runtime.common.files.RuntimePathProvider;
+
+import javax.inject.Inject;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Build the launch command for Java REEF processes for Azure Batch Windows pools.
+ */
+@Private
+public class WindowsCommandBuilder extends AbstractCommandBuilder {
+
+ private static final Class LAUNCHER_CLASS = REEFLauncher.class;
+ private static final Class SHIM_LAUNCHER_CLASS = EvaluatorShimLauncher.class;
+ private static final List<String> COMMAND_LIST_PREFIX = Collections.unmodifiableList(
+ Arrays.asList(
+ "Add-Type -AssemblyName System.IO.Compression.FileSystem; ",
+ "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\" +
+ AzureBatchFileNames.getTaskJarFileName() + "\\\", " +
+ "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\"); ")
+ );
+ private static final char CLASSPATH_SEPARATOR_CHAR = ';';
+ private static final String OS_COMMAND_FORMAT = "powershell.exe /c \"%s\";";
+
+ @Inject
+ WindowsCommandBuilder(
+ final ClasspathProvider classpathProvider,
+ final RuntimePathProvider runtimePathProvider,
+ final REEFFileNames reefFileNames,
+ final AzureBatchFileNames azureBatchFileNames) {
+ super(LAUNCHER_CLASS, SHIM_LAUNCHER_CLASS, COMMAND_LIST_PREFIX, OS_COMMAND_FORMAT,
+ classpathProvider, runtimePathProvider, reefFileNames, azureBatchFileNames);
+ }
+
+ @Override
+ protected String getDriverClasspath() {
+ return String.format("'%s'", StringUtils.join(
+ super.classpathProvider.getDriverClasspath(), CLASSPATH_SEPARATOR_CHAR));
+ }
+
+ @Override
+ protected String getEvaluatorShimClasspath() {
+ return String.format("'%s'", StringUtils.join(
+ super.classpathProvider.getEvaluatorClasspath(), CLASSPATH_SEPARATOR_CHAR));
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java
new file mode 100644
index 0000000..edd04b9
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/command/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Command line utilities for Azure Batch runtime.
+ */
+package org.apache.reef.runtime.azbatch.util.command;
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java
new file mode 100644
index 0000000..622e972
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * A REEF runtime for Azure Batch.
+ */
+package org.apache.reef.runtime.azbatch.util;
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java
new file mode 100644
index 0000000..83b522b
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/AzureStorageClient.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.storage;
+
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.blob.*;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageBlobSASTokenValidityHours;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageContainerName;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Azure storage utility to upload Driver and Evaluator jars to blob storage
+ * and generate SAS URIs.
+ */
+@Private
+public final class AzureStorageClient {
+ private static final Logger LOG = Logger.getLogger(AzureStorageClient.class.getName());
+
+ private final ICloudBlobClientProvider cloudBlobClientProvider;
+
+ private final String azureStorageContainerName;
+ private final int blobSASTokenValidityHours;
+
+ @Inject
+ AzureStorageClient(
+ final ICloudBlobClientProvider cloudBlobClientProvider,
+ @Parameter(AzureStorageContainerName.class) final String azureStorageContainerName,
+ @Parameter(AzureStorageBlobSASTokenValidityHours.class) final int blobSASTokenValidityHours) {
+ this.cloudBlobClientProvider = cloudBlobClientProvider;
+ this.azureStorageContainerName = azureStorageContainerName;
+ this.blobSASTokenValidityHours = blobSASTokenValidityHours;
+ }
+
+ public URI getJobSubmissionFolderUri(final String jobFolder) throws IOException {
+ final CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient();
+ try {
+ final CloudBlobContainer container = cloudBlobClient.getContainerReference(this.azureStorageContainerName);
+ return container.getDirectoryReference(jobFolder).getUri();
+ } catch (URISyntaxException | StorageException e) {
+ throw new IOException("Failed to get the job submission folder URI", e);
+ }
+ }
+
+
+ public String createContainerSharedAccessSignature() throws IOException {
+ try {
+ CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient();
+ CloudBlobContainer cloudBlobContainer = cloudBlobClient.getContainerReference(this.azureStorageContainerName);
+ cloudBlobContainer.createIfNotExists();
+
+ return cloudBlobContainer.generateSharedAccessSignature(getSharedAccessContainerPolicy(), null);
+
+ } catch (StorageException | URISyntaxException | InvalidKeyException e) {
+ throw new IOException("Failed to generate a shared access signature for storage container.", e);
+ }
+ }
+
+
+ /**
+ * Upload a file to the storage account.
+ *
+ * @param jobFolder the path to the destination folder within storage container.
+ * @param file the source file.
+ * @return the SAS URI to the uploaded file.
+ * @throws IOException
+ */
+ public URI uploadFile(final String jobFolder, final File file) throws IOException {
+
+ LOG.log(Level.INFO, "Uploading [{0}] to [{1}]", new Object[]{file, jobFolder});
+
+ try {
+ final CloudBlobClient cloudBlobClient = this.cloudBlobClientProvider.getCloudBlobClient();
+ final CloudBlobContainer container = cloudBlobClient.getContainerReference(this.azureStorageContainerName);
+
+ final String destination = String.format("%s/%s", jobFolder, file.getName());
+ final CloudBlockBlob blob = container.getBlockBlobReference(destination);
+
+ try (FileInputStream fis = new FileInputStream(file)) {
+ blob.upload(fis, file.length());
+ }
+
+ LOG.log(Level.FINE, "Uploaded to: {0}", blob.getStorageUri().getPrimaryUri());
+ return this.cloudBlobClientProvider.generateSharedAccessSignature(blob, getSharedAccessBlobPolicy());
+
+ } catch (final URISyntaxException | StorageException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private SharedAccessBlobPolicy getSharedAccessBlobPolicy() {
+ return getSharedAccessBlobPolicy(EnumSet.of(SharedAccessBlobPermissions.READ));
+ }
+
+ private SharedAccessBlobPolicy getSharedAccessContainerPolicy() {
+ return getSharedAccessBlobPolicy(EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.WRITE));
+ }
+
+ private SharedAccessBlobPolicy getSharedAccessBlobPolicy(
+ final EnumSet<SharedAccessBlobPermissions> permissions) {
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.add(Calendar.HOUR, this.blobSASTokenValidityHours);
+ Date tokenExpirationDate = calendar.getTime();
+
+ final SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
+ policy.setPermissions(permissions);
+ policy.setSharedAccessStartTime(Calendar.getInstance().getTime());
+ policy.setSharedAccessExpiryTime(tokenExpirationDate);
+
+ return policy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java
new file mode 100644
index 0000000..d51dbc2
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/ICloudBlobClientProvider.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.storage;
+
+import com.microsoft.windowsazure.storage.blob.CloudBlob;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * An interface for classes that provide an instance of {@link CloudBlobClient} based
+ * on available authentication mechanism.
+ */
+public interface ICloudBlobClientProvider {
+
+ /**
+ * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism.
+ * @return an instance of {@link CloudBlobClient}.
+ * @throws IOException
+ */
+ CloudBlobClient getCloudBlobClient() throws IOException;
+
+ /**
+ * Generates a Shared Access Key URI for the given {@link CloudBlob}.
+ * @param cloudBlob cloud blob to create a Shared Access Key URI for.
+ * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's
+ * validity time period.
+ * @return a Shared Access Key URI for the given {@link CloudBlob}.
+ * @throws IOException
+ */
+ URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java
new file mode 100644
index 0000000..e678445
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/SharedAccessSignatureCloudBlobClientProvider.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.storage;
+
+import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.blob.CloudBlob;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy;
+import com.microsoft.windowsazure.storage.core.PathUtility;
+import com.microsoft.windowsazure.storage.core.UriQueryBuilder;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Cloud Blob client provider that uses Azure Storage Shared Access Signature authorization.
+ */
+public final class SharedAccessSignatureCloudBlobClientProvider implements ICloudBlobClientProvider {
+
+ private static final Logger LOG = Logger.getLogger(AzureStorageClient.class.getName());
+
+ public static final String AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV = "AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV";
+
+ private static final String AZURE_STORAGE_ACCOUNT_URI_FORMAT = "https://%s.blob.core.windows.net";
+
+ private final String azureStorageAccountName;
+ private final String azureStorageContainerSASToken;
+
+ @Inject
+ SharedAccessSignatureCloudBlobClientProvider(
+ @Parameter(AzureStorageAccountName.class) final String azureStorageAccountName) {
+ this.azureStorageAccountName = azureStorageAccountName;
+ this.azureStorageContainerSASToken = System.getenv(AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV);
+ }
+
+ /**
+ * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism.
+ * @return an instance of {@link CloudBlobClient}.
+ * @throws IOException
+ */
+ @Override
+ public CloudBlobClient getCloudBlobClient() throws IOException {
+ StorageCredentialsSharedAccessSignature signature =
+ new StorageCredentialsSharedAccessSignature(this.azureStorageContainerSASToken);
+ URI storageAccountUri;
+ try {
+ storageAccountUri = new URI(String.format(AZURE_STORAGE_ACCOUNT_URI_FORMAT, this.azureStorageAccountName));
+ } catch (URISyntaxException e) {
+ throw new IOException("Failed to generate Storage Account URI", e);
+ }
+
+ return new CloudBlobClient(storageAccountUri, signature);
+ }
+
+ /**
+ * Generates a Shared Access Key URI for the given {@link CloudBlob}.
+ * @param cloudBlob cloud blob to create a Shared Access Key URI for.
+ * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's
+ * validity time period.
+ * @return a Shared Access Key URI for the given {@link CloudBlob}.
+ * @throws IOException
+ */
+ @Override
+ public URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy)
+ throws IOException {
+ try {
+ final URI uri = cloudBlob.getStorageUri().getPrimaryUri();
+
+ Map<String, String[]> queryString = PathUtility.parseQueryString(this.azureStorageContainerSASToken);
+
+ UriQueryBuilder builder = new UriQueryBuilder();
+ for (Map.Entry<String, String[]> entry : queryString.entrySet()) {
+ for (String value : entry.getValue()) {
+ builder.add(entry.getKey(), value);
+ }
+ }
+
+ URI result = builder.addToURI(uri);
+ LOG.log(Level.INFO, "Here's the URI: " + result);
+
+ return result;
+ } catch (StorageException | URISyntaxException e) {
+ throw new IOException("Failed to generated a Shared Access Signature.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java
new file mode 100644
index 0000000..ae8613d
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/StorageKeyCloudBlobProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch.util.storage;
+
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.blob.CloudBlob;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountKey;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+
+/**
+ * Cloud Blob client provider that uses Azure Storage Shared Key authorization.
+ */
+@Private
+public final class StorageKeyCloudBlobProvider implements ICloudBlobClientProvider {
+
+ private static final String AZURE_STORAGE_CONNECTION_STRING_FORMAT =
+ "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s";
+
+ private final String azureStorageAccountName;
+ private final String azureStorageAccountKey;
+
+ @Inject
+ StorageKeyCloudBlobProvider(
+ @Parameter(AzureStorageAccountName.class) final String azureStorageAccountName,
+ @Parameter(AzureStorageAccountKey.class) final String azureStorageAccountKey) {
+ this.azureStorageAccountName = azureStorageAccountName;
+ this.azureStorageAccountKey = azureStorageAccountKey;
+ }
+
+ /**
+ * Returns an instance of {@link CloudBlobClient} based on available authentication mechanism.
+ * @return an instance of {@link CloudBlobClient}.
+ * @throws IOException
+ */
+ @Override
+ public CloudBlobClient getCloudBlobClient() throws IOException {
+ String connectionString = String.format(AZURE_STORAGE_CONNECTION_STRING_FORMAT,
+ this.azureStorageAccountName, this.azureStorageAccountKey);
+ try {
+ return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
+ } catch (URISyntaxException | InvalidKeyException e) {
+ throw new IOException("Failed to create a Cloud Storage Account.", e);
+ }
+ }
+
+ /**
+ * Generates a Shared Access Key URI for the given {@link CloudBlob}.
+ * @param cloudBlob cloud blob to create a Shared Access Key URI for.
+ * @param policy an instance of {@link SharedAccessBlobPolicy} that specifies permissions and signature's
+ * validity time period.
+ * @return a Shared Access Key URI for the given {@link CloudBlob}.
+ * @throws IOException
+ */
+ @Override
+ public URI generateSharedAccessSignature(final CloudBlob cloudBlob, final SharedAccessBlobPolicy policy)
+ throws IOException {
+ try {
+ final String sas = cloudBlob.generateSharedAccessSignature(policy, null);
+ final String uri = cloudBlob.getStorageUri().getPrimaryUri().toString();
+ return new URI(uri + "?" + sas);
+ } catch (StorageException | InvalidKeyException | URISyntaxException e) {
+ throw new IOException("Failed to generated a Shared Access Signature.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java
new file mode 100644
index 0000000..adbe05f
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/main/java/org/apache/reef/runtime/azbatch/util/storage/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Azure Storage utilities for Azure Batch runtime.
+ */
+package org.apache.reef.runtime.azbatch.util.storage;