You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/01/24 14:42:09 UTC
spark git commit: [SPARK-14049][CORE] Add functionality in spark
history sever API to query applications by end time
Repository: spark
Updated Branches:
refs/heads/master 752502be0 -> 0ff67a1cf
[SPARK-14049][CORE] Add functionality in spark history sever API to query applications by end time
## What changes were proposed in this pull request?
Currently, spark history server REST API provides functionality to query applications by application start time range based on minDate and maxDate query parameters, but it lacks support to query applications by their end time. In this pull request we are proposing optional minEndDate and maxEndDate query parameters and filtering capability based on these parameters to spark history server REST API. This functionality can be used for following queries,
1. Applications finished in last 'x' minutes
2. Applications finished before 'y' time
3. Applications finished between 'x' time to 'y' time
4. Applications started from 'x' time and finished before 'y' time.
For backward compatibility, we can keep existing minDate and maxDate query parameters as they are and they can continue support filtering based on start time range.
## How was this patch tested?
Existing unit tests and 4 new unit tests.
Author: Parag Chaudhari <pa...@amazon.com>
Closes #11867 from paragpc/master-SHS-query-by-endtime_2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0ff67a1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0ff67a1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0ff67a1c
Branch: refs/heads/master
Commit: 0ff67a1cf91ce4a36657c789c0fe676f4f89282f
Parents: 752502b
Author: Parag Chaudhari <pa...@amazon.com>
Authored: Tue Jan 24 08:41:46 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Tue Jan 24 08:41:46 2017 -0600
----------------------------------------------------------------------
.../status/api/v1/ApplicationListResource.scala | 22 ++++-
.../maxEndDate_app_list_json_expectation.json | 95 ++++++++++++++++++++
...nd_maxEndDate_app_list_json_expectation.json | 53 +++++++++++
...nd_maxEndDate_app_list_json_expectation.json | 53 +++++++++++
.../minEndDate_app_list_json_expectation.json | 70 +++++++++++++++
.../deploy/history/HistoryServerSuite.scala | 6 ++
docs/monitoring.md | 17 +++-
7 files changed, 311 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
index 7677929..a023926 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -30,6 +30,8 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
@QueryParam("status") status: JList[ApplicationStatus],
@DefaultValue("2010-01-01") @QueryParam("minDate") minDate: SimpleDateParam,
@DefaultValue("3000-01-01") @QueryParam("maxDate") maxDate: SimpleDateParam,
+ @DefaultValue("2010-01-01") @QueryParam("minEndDate") minEndDate: SimpleDateParam,
+ @DefaultValue("3000-01-01") @QueryParam("maxEndDate") maxEndDate: SimpleDateParam,
@QueryParam("limit") limit: Integer)
: Iterator[ApplicationInfo] = {
@@ -43,11 +45,27 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
// keep the app if *any* attempts fall in the right time window
((!anyRunning && includeCompleted) || (anyRunning && includeRunning)) &&
app.attempts.exists { attempt =>
- val start = attempt.startTime.getTime
- start >= minDate.timestamp && start <= maxDate.timestamp
+ isAttemptInRange(attempt, minDate, maxDate, minEndDate, maxEndDate, anyRunning)
}
}.take(numApps)
}
+
+ private def isAttemptInRange(
+ attempt: ApplicationAttemptInfo,
+ minStartDate: SimpleDateParam,
+ maxStartDate: SimpleDateParam,
+ minEndDate: SimpleDateParam,
+ maxEndDate: SimpleDateParam,
+ anyRunning: Boolean): Boolean = {
+ val startTimeOk = attempt.startTime.getTime >= minStartDate.timestamp &&
+ attempt.startTime.getTime <= maxStartDate.timestamp
+ // If the maxEndDate is in the past, exclude all running apps.
+ val endTimeOkForRunning = anyRunning && (maxEndDate.timestamp > System.currentTimeMillis())
+ val endTimeOkForCompleted = !anyRunning && (attempt.endTime.getTime >= minEndDate.timestamp &&
+ attempt.endTime.getTime <= maxEndDate.timestamp)
+ val endTimeOk = endTimeOkForRunning || endTimeOkForCompleted
+ startTimeOk && endTimeOk
+ }
}
private[spark] object ApplicationsListResource {
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
new file mode 100644
index 0000000..1842f18
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
@@ -0,0 +1,95 @@
+[ {
+ "id" : "local-1430917381535",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:00.950GMT",
+ "lastUpdated" : "",
+ "duration" : 57,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-05-06T13:03:00.880GMT",
+ "endTime" : "2015-05-06T13:03:00.890GMT",
+ "lastUpdated" : "",
+ "duration" : 10,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890
+ } ]
+}, {
+ "id" : "local-1426533911241",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-03-17T23:11:50.242GMT",
+ "endTime" : "2015-03-17T23:12:25.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-03-16T19:25:10.242GMT",
+ "endTime" : "2015-03-16T19:25:45.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177
+ } ]
+}, {
+ "id" : "local-1425081759269",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-02-28T00:02:38.277GMT",
+ "endTime" : "2015-02-28T00:02:46.912GMT",
+ "lastUpdated" : "",
+ "duration" : 8635,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912
+ } ]
+}, {
+ "id" : "local-1422981780767",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-02-03T16:42:59.720GMT",
+ "endTime" : "2015-02-03T16:43:08.731GMT",
+ "lastUpdated" : "",
+ "duration" : 9011,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731
+ } ]
+}, {
+ "id" : "local-1422981759269",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-02-03T16:42:38.277GMT",
+ "endTime" : "2015-02-03T16:42:46.912GMT",
+ "lastUpdated" : "",
+ "duration" : 8635,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912
+ } ]
+} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
new file mode 100644
index 0000000..24f9f21
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
@@ -0,0 +1,53 @@
+[ {
+ "id" : "local-1430917381535",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:00.950GMT",
+ "lastUpdated" : "",
+ "duration" : 57,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-05-06T13:03:00.880GMT",
+ "endTime" : "2015-05-06T13:03:00.890GMT",
+ "lastUpdated" : "",
+ "duration" : 10,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890
+ } ]
+}, {
+ "id" : "local-1426533911241",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-03-17T23:11:50.242GMT",
+ "endTime" : "2015-03-17T23:12:25.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-03-16T19:25:10.242GMT",
+ "endTime" : "2015-03-16T19:25:45.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177
+ } ]
+} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
new file mode 100644
index 0000000..3745e8a
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
@@ -0,0 +1,53 @@
+[ {
+ "id" : "local-1430917381535",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:00.950GMT",
+ "lastUpdated" : "",
+ "duration" : 57,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-05-06T13:03:00.880GMT",
+ "endTime" : "2015-05-06T13:03:00.890GMT",
+ "lastUpdated" : "",
+ "duration" : 10,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890
+ } ]
+}, {
+ "id" : "local-1426533911241",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-03-17T23:11:50.242GMT",
+ "endTime" : "2015-03-17T23:12:25.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-03-16T19:25:10.242GMT",
+ "endTime" : "2015-03-16T19:25:45.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177
+ } ]
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
new file mode 100644
index 0000000..05233db
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
@@ -0,0 +1,70 @@
+[ {
+ "id" : "app-20161116163331-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-16T22:33:29.916GMT",
+ "endTime" : "2016-11-16T22:33:40.587GMT",
+ "lastUpdated" : "",
+ "duration" : 10671,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "startTimeEpoch" : 1479335609916,
+ "lastUpdatedEpoch" : 0,
+ "endTimeEpoch" : 1479335620587
+ } ]
+}, {
+ "id" : "app-20161115172038-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-15T23:20:37.079GMT",
+ "endTime" : "2016-11-15T23:22:18.874GMT",
+ "lastUpdated" : "",
+ "duration" : 101795,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "startTimeEpoch" : 1479252037079,
+ "lastUpdatedEpoch" : 0,
+ "endTimeEpoch" : 1479252138874
+ } ]
+}, {
+ "id" : "local-1430917381534",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:11.398GMT",
+ "lastUpdated" : "",
+ "duration" : 10505,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398
+ } ]
+}, {
+ "id" : "local-1430917381535",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:00.950GMT",
+ "lastUpdated" : "",
+ "duration" : 57,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-05-06T13:03:00.880GMT",
+ "endTime" : "2015-05-06T13:03:00.890GMT",
+ "lastUpdated" : "",
+ "duration" : 10,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "completed" : true,
+ "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890
+ } ]
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index cd3959a..b2eded4 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -102,6 +102,12 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
"minDate app list json" -> "applications?minDate=2015-02-10",
"maxDate app list json" -> "applications?maxDate=2015-02-10",
"maxDate2 app list json" -> "applications?maxDate=2015-02-03T16:42:40.000GMT",
+ "minEndDate app list json" -> "applications?minEndDate=2015-05-06T13:03:00.950GMT",
+ "maxEndDate app list json" -> "applications?maxEndDate=2015-05-06T13:03:00.950GMT",
+ "minEndDate and maxEndDate app list json" ->
+ "applications?minEndDate=2015-03-16&maxEndDate=2015-05-06T13:03:00.950GMT",
+ "minDate and maxEndDate app list json" ->
+ "applications?minDate=2015-03-16&maxEndDate=2015-05-06T13:03:00.950GMT",
"limit app list json" -> "applications?limit=3",
"one app json" -> "applications/local-1422981780767",
"one app multi-attempt json" -> "applications/local-1426533911241",
http://git-wip-us.apache.org/repos/asf/spark/blob/0ff67a1c/docs/monitoring.md
----------------------------------------------------------------------
diff --git a/docs/monitoring.md b/docs/monitoring.md
index e918174..7ba4824 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -267,12 +267,23 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
<br>
<code>?status=[completed|running]</code> list only applications in the chosen state.
<br>
- <code>?minDate=[date]</code> earliest date/time to list.
+ <code>?minDate=[date]</code> earliest start date/time to list.
+ <br>
+ <code>?maxDate=[date]</code> latest start date/time to list.
+ <br>
+ <code>?minEndDate=[date]</code> earliest end date/time to list.
+ <br>
+ <code>?maxEndDate=[date]</code> latest end date/time to list.
+ <br>
+ <code>?limit=[limit]</code> limits the number of applications listed.
<br>Examples:
<br><code>?minDate=2015-02-10</code>
<br><code>?minDate=2015-02-03T16:42:40.000GMT</code>
- <br><code>?maxDate=[date]</code> latest date/time to list; uses same format as <code>minDate</code>.
- <br><code>?limit=[limit]</code> limits the number of applications listed.</td>
+ <br><code>?maxDate=2015-02-11T20:41:30.000GMT</code>
+ <br><code>?minEndDate=2015-02-12</code>
+ <br><code>?minEndDate=2015-02-12T09:15:10.000GMT</code>
+ <br><code>?maxEndDate=2015-02-14T16:30:45.000GMT</code>
+ <br><code>?limit=10</code></td>
</tr>
<tr>
<td><code>/applications/[app-id]/jobs</code></td>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org