[5/9] reef git commit: [REEF-1965] Implement REEF runtime for Azure Batch

[do...@apache.org]

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java
new file mode 100644
index 0000000..230cca3
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/CommandBuilderTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.azbatch;
+
+import org.apache.reef.runtime.azbatch.util.command.LinuxCommandBuilder;
+import org.apache.reef.runtime.azbatch.util.command.WindowsCommandBuilder;
+import org.apache.reef.runtime.common.client.api.JobSubmissionEvent;
+import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
+import org.apache.reef.tang.Injector;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.exceptions.InjectionException;
+import org.apache.reef.util.Optional;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for the CommandBuilder functions.
+ */
+public final class CommandBuilderTests {
+
+  private Injector injector;
+  private LinuxCommandBuilder linuxCommandBuilder;
+  private WindowsCommandBuilder windowsCommandBuilder;
+
+  @SuppressWarnings("unchecked")
+  @Before
+  public void setUp() throws InjectionException {
+    this.injector = Tang.Factory.getTang().newInjector();
+    RuntimeClasspathProvider classpathProvider = mock(RuntimeClasspathProvider.class);
+    when(classpathProvider.getDriverClasspathPrefix()).thenReturn(Arrays.asList("c:\\driverpath1", "c:\\driverpath2"));
+    when(classpathProvider.getEvaluatorClasspathPrefix())
+        .thenReturn(Arrays.asList("c:\\evaluatorpath1", "c:\\evaluatorpath2"));
+    when(classpathProvider.getDriverClasspathSuffix()).thenReturn(Arrays.asList("driverclasspathsuffix"));
+    when(classpathProvider.getEvaluatorClasspathSuffix()).thenReturn(Arrays.asList("evaluatorclasspathsuffix"));
+    this.injector
+        .bindVolatileInstance(RuntimeClasspathProvider.class, classpathProvider);
+    this.linuxCommandBuilder = this.injector.getInstance(LinuxCommandBuilder.class);
+    this.windowsCommandBuilder = this.injector.getInstance(WindowsCommandBuilder.class);
+
+  }
+
+  @Test
+  public void linuxCommandBuilderDriverTest() {
+    JobSubmissionEvent event = mock(JobSubmissionEvent.class);
+
+    Optional<Integer> memory = Optional.of(100);
+    when(event.getDriverMemory()).thenReturn(memory);
+
+    String actual = this.linuxCommandBuilder.buildDriverCommand(event);
+    String expected =
+        "/bin/sh -c \"unzip local.jar -d 'reef/'; {{JAVA_HOME}}/bin/java -Xmx100m -XX:PermSize=128m " +
+            "-XX:MaxPermSize=128m -ea -classpath " +
+            "c:\\driverpath1:c:\\driverpath2:reef/local/*:reef/global/*:driverclasspathsuffix " +
+            "-Dproc_reef org.apache.reef.runtime.common.REEFLauncher reef/local/driver.conf\"";
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test
+  public void windowsCommandBuilderDriverTest() {
+    JobSubmissionEvent event = mock(JobSubmissionEvent.class);
+
+    Optional<Integer> memory = Optional.of(100);
+    when(event.getDriverMemory()).thenReturn(memory);
+
+    String actual = this.windowsCommandBuilder.buildDriverCommand(event);
+    String expected = "powershell.exe /c \"Add-Type -AssemblyName System.IO.Compression.FileSystem;  " +
+        "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\local.jar\\\", " +
+        "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\");  {{JAVA_HOME}}/bin/java -Xmx100m -XX:PermSize=128m " +
+        "-XX:MaxPermSize=128m -ea -classpath " +
+        "'c:\\driverpath1;c:\\driverpath2;reef/local/*;reef/global/*;driverclasspathsuffix' " +
+        "-Dproc_reef org.apache.reef.runtime.common.REEFLauncher reef/local/driver.conf\";";
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test
+  public void linuxCommandBuilderShimEvaluatorTest() {
+    String actual = this.linuxCommandBuilder.buildEvaluatorShimCommand(1, "conf");
+    String expected = "/bin/sh -c \"unzip local.jar -d 'reef/'; {{JAVA_HOME}}/bin/java -Xmx1m " +
+        "-XX:PermSize=128m -XX:MaxPermSize=128m -ea " +
+        "-classpath c:\\evaluatorpath1:c:\\evaluatorpath2:reef/local/*:reef/global/*:evaluatorclasspathsuffix " +
+        "-Dproc_reef org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher conf\"";
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test
+  public void windowsCommandBuilderShimEvaluatorTest() {
+    String actual = this.windowsCommandBuilder.buildEvaluatorShimCommand(1, "conf");
+    String expected = "powershell.exe /c \"Add-Type -AssemblyName System.IO.Compression.FileSystem;  " +
+        "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\local.jar\\\", " +
+        "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\");  {{JAVA_HOME}}/bin/java -Xmx1m -XX:PermSize=128m " +
+        "-XX:MaxPermSize=128m -ea -classpath " +
+        "'c:\\evaluatorpath1;c:\\evaluatorpath2;reef/local/*;reef/global/*;evaluatorclasspathsuffix' -Dproc_reef " +
+        "org.apache.reef.runtime.azbatch.evaluator.EvaluatorShimLauncher conf\";";
+    Assert.assertEquals(expected, actual);
+  }
+}

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java
new file mode 100644
index 0000000..13a5caa
--- /dev/null
+++ b/lang/java/reef-runtime-azbatch/src/test/java/org/apache/reef/runtime/azbatch/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Unit tests for REEF Driver running under Azure Batch runtime.
+ */
+package org.apache.reef.runtime.azbatch;

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/reef-tests/pom.xml b/lang/java/reef-tests/pom.xml
index beefd5f..b096e39 100644
--- a/lang/java/reef-tests/pom.xml
+++ b/lang/java/reef-tests/pom.xml
@@ -44,6 +44,11 @@ under the License.
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
+            <artifactId>reef-runtime-azbatch</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
             <artifactId>reef-runtime-local</artifactId>
             <version>${project.version}</version>
         </dependency>

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java
new file mode 100644
index 0000000..7076dc9
--- /dev/null
+++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AzureBatchTestEnvironment.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.tests;
+
+import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration;
+import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationProvider;
+import org.apache.reef.runtime.azbatch.driver.RuntimeIdentifier;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Injector;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.exceptions.InjectionException;
+
+import java.io.IOException;
+
+/**
+ * A TestEnvironment for the Azure Batch resourcemanager.
+ */
+public final class AzureBatchTestEnvironment extends TestEnvironmentBase implements TestEnvironment {
+
+  // Used to make sure the tests call the methods in the right order.
+  private boolean ready = false;
+
+  @Override
+  public synchronized void setUp() {
+    this.ready = true;
+  }
+
+  @Override
+  public synchronized Configuration getRuntimeConfiguration() {
+    assert this.ready;
+    try {
+      Configuration userConfiguration = AzureBatchRuntimeConfiguration.fromEnvironment();
+      final Injector injector = Tang.Factory.getTang().newInjector(userConfiguration);
+      final AzureBatchRuntimeConfigurationProvider runtimeConfigurationProvider =
+          injector.getInstance(AzureBatchRuntimeConfigurationProvider.class);
+      return runtimeConfigurationProvider.getAzureBatchRuntimeConfiguration();
+    } catch (IOException | InjectionException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public synchronized void tearDown() {
+    assert this.ready;
+    this.ready = false;
+  }
+
+  @Override
+  public int getTestTimeout() {
+    return 5 * 60000; // 5 min.
+  }
+
+  @Override
+  public String getRuntimeName() {
+    return RuntimeIdentifier.RUNTIME_NAME;
+  }
+}

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java
index ab1454b..f716bec 100644
--- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java
+++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironmentFactory.java
@@ -37,8 +37,9 @@ public final class TestEnvironmentFactory {
   public static TestEnvironment getNewTestEnvironment() {
     final boolean isYarn = Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"));
     final boolean isMesos = Boolean.parseBoolean(System.getenv("REEF_TEST_MESOS"));
+    final boolean isAzBatch = Boolean.parseBoolean(System.getenv("REEF_TEST_AZBATCH"));
 
-    if (isYarn && isMesos) {
+    if (isYarn ? (isMesos || isAzBatch) : (isMesos && isAzBatch)) {
       throw new RuntimeException("Cannot test on two runtimes at once");
     } else if (isYarn) {
       LOG.log(Level.INFO, "Running tests on YARN");
@@ -46,6 +47,9 @@ public final class TestEnvironmentFactory {
     } else if (isMesos) {
       LOG.log(Level.INFO, "Running tests on Mesos");
       return new MesosTestEnvironment();
+    } else if (isAzBatch) {
+      LOG.log(Level.INFO, "Running tests on Azure Batch");
+      return new AzureBatchTestEnvironment();
     } else {
       LOG.log(Level.INFO, "Running tests on Local");
       return new LocalTestEnvironment();

http://git-wip-us.apache.org/repos/asf/reef/blob/561a336f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7d4431e..8d58a93 100644
--- a/pom.xml
+++ b/pom.xml
@@ -753,6 +753,13 @@ under the License.
                 <artifactId>mesos</artifactId>
                 <version>0.25.0</version>
             </dependency>
+
+            <!-- Microsoft Azure Batch APIs -->
+            <dependency>
+                <groupId>com.microsoft.azure</groupId>
+                <artifactId>azure-batch</artifactId>
+                <version>3.0.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -770,6 +777,7 @@ under the License.
         <module>lang/java/reef-experimental</module>
         <module>lang/java/reef-io</module>
         <module>lang/java/reef-poison</module>
+        <module>lang/java/reef-runtime-azbatch</module>
         <module>lang/java/reef-runtime-hdinsight</module>
         <module>lang/java/reef-runtime-local</module>
         <module>lang/java/reef-runtime-yarn</module>