You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/10/29 23:22:26 UTC
svn commit: r1536919 - in /hive/trunk/hcatalog:
src/test/e2e/templeton/tests/jobstatus.conf
webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Author: thejas
Date: Tue Oct 29 22:22:26 2013
New Revision: 1536919
URL: http://svn.apache.org/r1536919
Log:
HIVE-5519 : Use paging mechanism for templeton get requests. (Hari Sankar Sivarama Subramaniyan via Thejas Nair)
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf?rev=1536919&r1=1536918&r2=1536919&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf Tue Oct 29 22:22:26 2013
@@ -152,6 +152,36 @@ $cfg =
'json_path' => {'$[-1:]' => 'job_.*'},
'status_code' => 200,
},
+ {
+ # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the oldest 2 jobs whose
+ # id is greater than job_0
+ 'num' => 9,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*&numrecords=2&jobid=job_0',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].id' => 'job_.*',
+ '$[-1:].detail.status.jobId' => 'job_.*',
+ '$[-1:].detail.status.runState' => '\\d+',
+ '$[-1:].detail.status.jobId' => 'job_.*',
+ '$[-1:].detail.status.jobComplete' => 'true',
+ '$[-1:].detail.profile.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.profile.jobFile' => '^.+$',
+ '$[-1:].detail.profile.url' => '^.+$',
+ '$[-1:].detail.profile.queueName' => '^.+$',
+ '$[-1:].detail.profile.jobID.id' => '\\d+',
+ '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+',
+ '$[-1:].detail.profile.jobId' => 'job_.*',
+ '$[-1:].detail.id' => 'job_.*',
+ '$[-1:].detail.parentId' => 'job_.*',
+ '$[-1:].detail.percentComplete' => '100%',
+ '$[-1:].detail.exitValue' => '0',
+ '$[-1:].detail.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.callback' => '^.+$',
+ '$[-1:].detail.completed' => 'done',
+ },
+ 'status_code' => 200,
+ },
]
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1536919&r1=1536918&r2=1536919&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Oct 29 22:22:26 2013
@@ -855,15 +855,59 @@ public class Server {
}
/**
- * Return all the known job ids for this user.
+ * Return all the known job ids for this user based on the optional filter conditions.
+ * <p>
+ * Example usages:
+ * 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan'
+ * Return all the Job IDs submitted by hsubramaniyan
+ * 2. curl -s
+ * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&showall=true'
+ * Return all the Job IDs that are visible to hsubramaniyan
+ * 3. curl -s
+ * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&jobid=job_201312091733_0003'
+ * Return all the Job IDs for hsubramaniyan after job_201312091733_0003.
+ * 4. curl -s 'http://localhost:50111/templeton/v1/jobs?
+ * user.name=hsubramaniyan&jobid=job_201312091733_0003&numrecords=5'
+ * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003.
+ * 5. curl -s
+ * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&numrecords=5'
+ * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list
+ * lexicographically.
+ * </p>
+ * <p>
+ * Supporting pagination using "jobid" and "numrecords" parameters:
+ * Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n
+ * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid"
+ * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's
+ * last record and get the Job Id of the last record as job_yyy_k, else quit.
+ * Step 4: set "jobid"=job_yyy_k and go to step 2.
+ * </p>
+ * @param fields If "fields" set to "*", the request will return full details of the job.
+ * If "fields" is missing, will only return the job ID. Currently the value can only
+ * be "*", other values are not allowed and will throw exception.
+ * @param showall If "showall" is set to "true", the request will return all jobs the user
+ * has permission to view, not only the jobs belonging to the user.
+ * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater
+ * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001",
+ * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of
+ * records returned depends on the value of "numrecords".
+ * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords
+ * records appearing after "jobid" will be returned after sorting the Job Id list
+ * lexicographically.
+ * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will
+ * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present
+ * and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are returned.
+ * @return list of job items based on the filter conditions specified by the user.
*/
@GET
@Path("jobs")
@Produces({MediaType.APPLICATION_JSON})
public List<JobItemBean> showJobList(@QueryParam("fields") String fields,
- @QueryParam("showall") boolean showall)
+ @QueryParam("showall") boolean showall,
+ @QueryParam("jobid") String jobid,
+ @QueryParam("numrecords") String numrecords)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
-
+
verifyUser();
boolean showDetails = false;
@@ -877,7 +921,46 @@ public class Server {
ListDelegator ld = new ListDelegator(appConf);
List<String> list = ld.run(getDoAsUser(), showall);
List<JobItemBean> detailList = new ArrayList<JobItemBean>();
+ int currRecord = 0;
+ int numRecords;
+
+ // Parse numrecords to an integer
+ try {
+ if (numrecords != null) {
+ numRecords = Integer.parseInt(numrecords);
+ if (numRecords <= 0) {
+ throw new BadParam("numrecords should be an integer > 0");
+ }
+ }
+ else {
+ numRecords = -1;
+ }
+ }
+ catch(Exception e) {
+ throw new BadParam("Invalid numrecords format: numrecords should be an integer > 0");
+ }
+
+ // Sort the list lexicographically
+ Collections.sort(list);
+
for (String job : list) {
+ // If numRecords = -1, fetch all records.
+ // Hence skip all the below checks when numRecords = -1.
+ if (numRecords != -1) {
+ // If currRecord >= numRecords, we have already fetched the top #numRecords
+ if (currRecord >= numRecords) {
+ break;
+ }
+ // If the current record needs to be returned based on the
+ // filter conditions specified by the user, increment the counter
+ else if ((jobid != null && job.compareTo(jobid) > 0) || jobid == null) {
+ currRecord++;
+ }
+ // The current record should not be included in the output detailList.
+ else {
+ continue;
+ }
+ }
JobItemBean jobItem = new JobItemBean();
jobItem.id = job;
if (showDetails) {