You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by sr...@apache.org on 2017/01/25 16:24:48 UTC
tez git commit: TEZ-3494. Support relative url for
tez-ui.history-url.base config (hitesh via sree)
Repository: tez
Updated Branches:
refs/heads/master d227b99a4 -> 24a8a95bd
TEZ-3494. Support relative url for tez-ui.history-url.base config (hitesh via sree)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/24a8a95b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/24a8a95b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/24a8a95b
Branch: refs/heads/master
Commit: 24a8a95bd2093377154760cf57ebe4f56c6197f4
Parents: d227b99
Author: Sreenath Somarajapuram <sr...@apache.org>
Authored: Wed Jan 25 21:55:20 2017 +0530
Committer: Sreenath Somarajapuram <sr...@apache.org>
Committed: Wed Jan 25 21:55:20 2017 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/api/TezConfiguration.java | 12 ++++++
.../tez/dag/app/rm/TaskSchedulerManager.java | 14 +++++--
.../dag/app/rm/TestTaskSchedulerManager.java | 42 +++++++++++++++-----
4 files changed, 56 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4d43bca..d6798ff 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -177,6 +177,7 @@ ALL CHANGES:
TEZ-3583. Tez UI: UTs are flaky because of a dependency issue
TEZ-3580. Tez UI: Pagination broken on queries page
TEZ-3575. RM have started forwarding origin. Use that in AMWebController for CORS support
+ TEZ-3494. Support relative url for tez-ui.history-url.base config
Release 0.8.5: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index e27cdf8..8747616 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1335,6 +1335,18 @@ public class TezConfiguration extends Configuration {
TEZ_PREFIX + "yarn.ats.max.events.per.batch";
public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5;
+ /**
+ * Boolean value. Default true.
+ * Whether to fix the history url if it has not been configured correctly i.e. it does not have a
+ * scheme in the value. By default, the url will be prepended with a scheme (http) if there is
+ * none present.
+ */
+ @Private
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED =
+ TEZ_PREFIX + "am.ui.history.url.scheme.check.enabled";
+ public static final boolean TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT = true;
+
/**
* Int value. Time, in milliseconds, to wait for an event before sending a batch to ATS.
http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
index e68c9b8..7989e5f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
@@ -988,9 +988,17 @@ public class TaskSchedulerManager extends AbstractService implements
.replaceAll(HISTORY_URL_BASE, historyUrlBase)
.replaceAll("([^:])/{2,}", "$1/");
- // make sure we have a valid scheme
- if (!historyUrl.startsWith("http")) {
- historyUrl = "http://" + historyUrl;
+ // FIXME: we should not need this check in the first place.
+ // Might be better to remove it in the next release and not fix badly configured URLs.
+ boolean checkSchemePrefix = config.getBoolean(
+ TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED,
+ TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT);
+
+ if (checkSchemePrefix) {
+ // make sure we have a valid scheme
+ if (!historyUrl.startsWith("http")) {
+ historyUrl = "http://" + historyUrl;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
index 791bb7f..66c209a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
@@ -394,33 +394,55 @@ public class TestTaskSchedulerManager {
// ensure history url is empty when timeline server is not the logging class
conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
- assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
+ assertEquals("http://ui-host:9999/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
// ensure the trailing / in history url is handled
conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
- assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
+ assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
// ensure missing scheme in history url is handled
conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
- Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
+ assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
// handle bad template ex without begining /
conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
"__HISTORY_URL_BASE__#/somepath");
- assertTrue("http://ui-host:9998/#/somepath"
- .equals(schedulerHandler.getHistoryUrl()));
+ assertEquals("http://ui-host:9998/#/somepath",
+ schedulerHandler.getHistoryUrl());
conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
"__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
- assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
+ assertEquals("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
}
+ @Test (timeout = 5000)
+ public void testHistoryUrlWithoutScheme() throws Exception {
+ Configuration conf = schedulerHandler.appContext.getAMConf();
+ final ApplicationId mockApplicationId = mock(ApplicationId.class);
+ doReturn("TEST_APP_ID").when(mockApplicationId).toString();
+ doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
+
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "/foo/bar/");
+ conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, false);
+ assertEquals("/foo/bar/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
+
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/");
+ assertEquals("ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
+
+ conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, true);
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/");
+ assertEquals("http://ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID",
+ schedulerHandler.getHistoryUrl());
+ }
+
@Test(timeout = 5000)
public void testNoSchedulerSpecified() throws IOException {
try {