You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2017/08/30 15:55:33 UTC
[7/9] incubator-gobblin git commit: Tests for Azkaban Orchestrator
Tests for Azkaban Orchestrator
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/ee3e5481
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/ee3e5481
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/ee3e5481
Branch: refs/heads/master
Commit: ee3e5481a1ef538fd5d5610e30254cb1fc80cad1
Parents: 0bb5139
Author: Abhishek Tiwari <ab...@gmail.com>
Authored: Wed Aug 30 04:43:20 2017 -0700
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Wed Aug 30 04:43:20 2017 -0700
----------------------------------------------------------------------
gobblin-modules/gobblin-azkaban/build.gradle | 3 +
.../orchestration/AzkabanAjaxAPIClient.java | 7 +-
.../orchestration/AzkabanProjectConfig.java | 11 +-
.../orchestration/AzkabanAjaxAPIClientTest.java | 97 +++++++++++++++++
.../orchestration/AzkabanProjectConfigTest.java | 109 +++++++++++++++++++
.../src/test/resources/reference.conf | 15 +++
6 files changed, 238 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/build.gradle b/gobblin-modules/gobblin-azkaban/build.gradle
index 4bebecc..2f1dde6 100644
--- a/gobblin-modules/gobblin-azkaban/build.gradle
+++ b/gobblin-modules/gobblin-azkaban/build.gradle
@@ -41,6 +41,9 @@ dependencies {
compile externalDependency.typesafeConfig
compile externalDependency.hadoopYarnApi
compile externalDependency.findBugsAnnotations
+
+ testCompile externalDependency.mockito
+ testCompile externalDependency.powermock
}
test {
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
index d0b8471..90bf005 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
@@ -51,6 +51,7 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.collect.Maps;
import com.google.gson.JsonElement;
@@ -345,14 +346,16 @@ public class AzkabanAjaxAPIClient {
return postRequest;
}
- private static Map<String, String> executeGetRequest(HttpGet getRequest) throws IOException {
+ @VisibleForTesting
+ protected static Map<String, String> executeGetRequest(HttpGet getRequest) throws IOException {
// Make the call, get response
@Cleanup CloseableHttpClient httpClient = getHttpClient();
HttpResponse response = httpClient.execute(getRequest);
return handleResponse(response);
}
- private static Map<String, String> executePostRequest(HttpPost postRequest) throws IOException {
+ @VisibleForTesting
+ protected static Map<String, String> executePostRequest(HttpPost postRequest) throws IOException {
// Make the call, get response
@Cleanup CloseableHttpClient httpClient = getHttpClient();
HttpResponse response = httpClient.execute(postRequest);
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
index 2bac65d..b99683d 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
@@ -24,6 +24,7 @@ import lombok.Builder;
import lombok.Getter;
import lombok.ToString;
+import org.apache.commons.lang.StringUtils;
import org.apache.gobblin.runtime.api.JobSpec;
import org.apache.gobblin.util.ConfigUtils;
@@ -76,8 +77,14 @@ public class AzkabanProjectConfig {
this.azkabanZipJarNames = Optional.ofNullable(ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_NAMES_KEY));
this.azkabanZipJarUrlTemplate = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_URL_TEMPLATE_KEY, null));
this.azkabanZipJarVersion = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_VERSION_KEY, null));
- this.azkabanZipAdditionalFiles = Optional.ofNullable(
- ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY));
+ if (config.hasPath(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY) &&
+ StringUtils.isNotBlank(config.getString(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY))) {
+ this.azkabanZipAdditionalFiles = Optional.ofNullable(
+ ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY));
+ } else {
+ this.azkabanZipAdditionalFiles = Optional.empty();
+ }
+
this.failIfJarNotFound = ConfigUtils.getBoolean(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_FAIL_IF_JARNOTFOUND_KEY, false);
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java
new file mode 100644
index 0000000..d7dda91
--- /dev/null
+++ b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+@Slf4j
+@Test(groups = { "org.apache.gobblin.service.modules.orchestration" })
+public class AzkabanAjaxAPIClientTest {
+
+ @Test
+ public void testCurrentTimeWithinWindow()
+ throws ParseException {
+ // Current hour
+ Calendar now = Calendar.getInstance();
+ int currentHour = now.get(Calendar.HOUR_OF_DAY);
+
+ // Generate a window encapsulating the current time
+ int windowStartInHours = currentHour < 2 ? 24 + (currentHour - 2) : currentHour - 2;
+ int windowEndInHours = (currentHour + 3) % 24;
+ int delayInMinutes = 0;
+
+ // Get computed scheduled time
+ String outputScheduledString =
+ AzkabanAjaxAPIClient.getScheduledTimeInAzkabanFormat(windowStartInHours, windowEndInHours, delayInMinutes);
+
+ // Verify that output schedule time is within window
+ Assert.assertTrue(isWithinWindow(windowStartInHours, windowEndInHours, outputScheduledString));
+ }
+
+ @Test
+ public void testCurrentTimeOutsideWindow()
+ throws ParseException {
+ // Current hour
+ Calendar now = Calendar.getInstance();
+ int currentHour = now.get(Calendar.HOUR_OF_DAY);
+
+ // Generate a window NOT encapsulating the current time
+ int windowStartInHours = currentHour > 10 ? 1 : 11;
+ int windowEndInHours = currentHour > 10 ? 6 : 16;
+ int delayInMinutes = 0;
+
+ // Get computed scheduled time
+ String outputScheduledString =
+ AzkabanAjaxAPIClient.getScheduledTimeInAzkabanFormat(windowStartInHours, windowEndInHours, delayInMinutes);
+
+ // Verify that output schedule time is within window
+ Assert.assertTrue(isWithinWindow(windowStartInHours, windowEndInHours, outputScheduledString));
+ }
+
+ private boolean isWithinWindow(int windowStartInHours, int windowEndInHours, String outputScheduledString)
+ throws ParseException {
+ Calendar windowStart = Calendar.getInstance();
+ windowStart.set(Calendar.HOUR_OF_DAY, windowStartInHours);
+ windowStart.set(Calendar.MINUTE, 0);
+ windowStart.set(Calendar.SECOND, 0);
+
+ Calendar windowEnd = Calendar.getInstance();
+ windowEnd.set(Calendar.HOUR_OF_DAY, windowEndInHours);
+ windowEnd.set(Calendar.MINUTE, 0);
+ windowEnd.set(Calendar.SECOND, 0);
+
+ Date outputDate = new SimpleDateFormat("hh,mm,a,z").parse(outputScheduledString);
+ Calendar receivedTime = Calendar.getInstance();
+ receivedTime.set(Calendar.HOUR_OF_DAY, Integer.parseInt(new SimpleDateFormat("HH").format(outputDate)));
+ receivedTime.set(Calendar.MINUTE, Integer.parseInt(new SimpleDateFormat("mm").format(outputDate)));
+
+ log.info("Window start time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(windowStart.getTime()));
+ log.info("Window end time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(windowEnd.getTime()));
+ log.info("Output time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(receivedTime.getTime()));
+
+ return receivedTime.after(windowStart) && receivedTime.before(windowEnd);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java
new file mode 100644
index 0000000..9e189ab
--- /dev/null
+++ b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.net.URI;
+import java.util.Properties;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.runtime.api.JobSpec;
+import org.apache.gobblin.util.ConfigUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+
+
+@Slf4j
+@Test(groups = { "org.apache.gobblin.service.modules.orchestration" })
+public class AzkabanProjectConfigTest {
+
+ @Test
+ public void testProjectNameDefault() throws Exception {
+ String expectedProjectName = "GobblinService__uri";
+
+ Properties properties = new Properties();
+ JobSpec jobSpec = new JobSpec(new URI("uri"), "0.0", "test job spec",
+ ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
+ AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
+
+ String actualProjectName = azkabanProjectConfig.getAzkabanProjectName();
+
+ Assert.assertEquals(actualProjectName, expectedProjectName);
+ }
+
+ @Test
+ public void testProjectNameWithConfig() throws Exception {
+ String expectedProjectName = "randomPrefix_http___localhost_8000_context";
+
+ Properties properties = new Properties();
+ properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefix");
+ JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context"), "0.0", "test job spec",
+ ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
+ AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
+
+ String actualProjectName = azkabanProjectConfig.getAzkabanProjectName();
+
+ Assert.assertEquals(actualProjectName, expectedProjectName);
+ }
+
+ @Test
+ public void testProjectNameWithReallyLongName() throws Exception {
+ String expectedProjectName = "randomPrefixWithReallyLongName_http___localhost_8000__55490420";
+
+ Properties properties = new Properties();
+ properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefixWithReallyLongName");
+ JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context/that-keeps-expanding-and-explanding"),
+ "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
+ AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
+
+ String actualProjectName = azkabanProjectConfig.getAzkabanProjectName();
+
+ Assert.assertEquals(actualProjectName, expectedProjectName);
+ }
+
+ @Test
+ public void testProjectZipFileName() throws Exception {
+ String expectedZipFileName = "randomPrefix_http___localhost_8000_context.zip";
+
+ Properties properties = new Properties();
+ properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefix");
+ JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context"), "0.0", "test job spec",
+ ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
+ AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
+
+ String actualZipFileName = azkabanProjectConfig.getAzkabanProjectZipFilename();
+
+ Assert.assertEquals(actualZipFileName, expectedZipFileName);
+ }
+
+ @Test
+ public void testProjectZipFileNameForLongName() throws Exception {
+ String expectedZipFileName = "randomPrefixWithReallyLongName_http___localhost_8000__55490420.zip";
+
+ Properties properties = new Properties();
+ properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefixWithReallyLongName");
+ JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context/that-keeps-expanding-and-explanding"),
+ "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
+ AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
+
+ String actualZipFileName = azkabanProjectConfig.getAzkabanProjectZipFilename();
+
+ Assert.assertEquals(actualZipFileName, expectedZipFileName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf b/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf
new file mode 100644
index 0000000..936b8a1
--- /dev/null
+++ b/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf
@@ -0,0 +1,15 @@
+# Sample configuration properties with default values
+
+# Cluster configuration properties
+gobblin.cluster.app.name=GobblinStandaloneCluster
+gobblin.cluster.email.notification.on.shutdown=false
+gobblin.cluster.helix.instance.max.retries=2
+gobblin.cluster.work.dir=/tmp/gobblin-cluster
+
+# Helix/Zookeeper configuration properties
+gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
+gobblin.cluster.zk.connection.string="localhost:2181"
+
+fs.uri="file:///"
+
+job.execinfo.server.enabled=false