You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2017/01/23 09:31:13 UTC
falcon git commit: FALCON-2260 Enhance extension List api to support
for user extensions
Repository: falcon
Updated Branches:
refs/heads/master 3c011688b -> c67556852
FALCON-2260 Enhance extension List api to support for user extensions
Author: sandeep <sa...@gmail.com>
Reviewers: @PraveenAdlakha
Closes #343 from sandeepSamudrala/FALCON-2260 and squashes the following commits:
5d9c151 [sandeep] Incorporated review comments
410f6a7 [sandeep] Incorporated review comments
7609139 [sandeep] Merge branch 'master' of https://github.com/apache/falcon into FALCON-2260
8f2a8d4 [sandeep] Corrected error messages
c949059 [sandeep] FALCON-2260 Enhance extension List api to support for user extensions
d317d3d [sandeep] Fixed tests
6617993 [sandeep] Incorporated review comments
46ad617 [sandeep] Incorporated review comments
c1189be [sandeep] Incorporated review comments
a70f5a9 [sandeep] FALCON-2259 Unregister an extension only if no extension jobs are dependant on the extension
7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command
26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon
b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon
194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon
e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon
250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/c6755685
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/c6755685
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/c6755685
Branch: refs/heads/master
Commit: c67556852cee96a91613df53228025adf40b9064
Parents: 3c01168
Author: sandeep <sa...@gmail.com>
Authored: Mon Jan 23 15:00:50 2017 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Mon Jan 23 15:00:50 2017 +0530
----------------------------------------------------------------------
.../apache/falcon/cli/FalconExtensionCLI.java | 5 +-
.../org/apache/falcon/client/FalconClient.java | 6 +--
.../falcon/resource/ExtensionJobList.java | 53 ++++----------------
.../falcon/persistence/ExtensionJobsBean.java | 2 +-
.../site/twiki/falconcli/ExtensionList.twiki | 4 +-
docs/src/site/twiki/restapi/ExtensionList.twiki | 45 ++---------------
.../extensions/jdbc/ExtensionMetaStore.java | 33 ++++++------
.../falcon/extensions/store/ExtensionStore.java | 12 -----
.../extensions/jdbc/ExtensionMetaStoreTest.java | 6 ++-
.../resource/AbstractExtensionManager.java | 16 ++++--
.../resource/proxy/ExtensionManagerProxy.java | 50 ++----------------
.../falcon/resource/ExtensionManagerIT.java | 2 -
.../org/apache/falcon/resource/TestContext.java | 3 --
13 files changed, 56 insertions(+), 181 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
----------------------------------------------------------------------
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
index 2101d65..15eb8d5 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
@@ -152,10 +152,7 @@ public class FalconExtensionCLI extends FalconCLI{
} else if (optionsList.contains(FalconCLIConstants.LIST_OPT)) {
validateRequiredParameter(extensionName, EXTENSION_NAME_OPT);
ExtensionJobList jobs = client.listExtensionJob(extensionName, doAsUser,
- commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT),
- commandLine.getOptionValue(FalconCLIConstants.OFFSET_OPT),
- commandLine.getOptionValue(FalconCLIConstants.NUM_RESULTS_OPT),
- commandLine.getOptionValue(FalconCLIConstants.FIELDS_OPT));
+ commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT));
result = jobs != null ? jobs.toString() : "No extension job (" + extensionName + ") found.";
} else if (optionsList.contains(INSTANCES_OPT)) {
validateRequiredParameter(jobName, JOB_NAME_OPT);
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index ab98f4a..aca83e3 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -1247,15 +1247,11 @@ public class FalconClient extends AbstractFalconClient {
}
public ExtensionJobList listExtensionJob(final String extensionName, final String doAsUser,
- final String sortOrder, final String offset,
- final String numResults, final String fields) {
+ final String sortOrder) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.LIST.path, extensionName)
.addQueryParam(DO_AS_OPT, doAsUser)
- .addQueryParam(FIELDS, fields)
.addQueryParam(SORT_ORDER, sortOrder)
- .addQueryParam(OFFSET, offset)
- .addQueryParam(NUM_RESULTS, numResults)
.call(ExtensionOperations.LIST);
return getResponse(ExtensionJobList.class, clientResponse);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
----------------------------------------------------------------------
diff --git a/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java b/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
index ca031b5..244ea66 100644
--- a/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
+++ b/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
@@ -33,10 +33,10 @@ import java.util.List;
public class ExtensionJobList {
@XmlElement
- public int numJobs;
+ int numJobs;
@XmlElementWrapper(name = "jobs")
- public List<JobElement> job;
+ public List<String> job;
public ExtensionJobList() {
numJobs = 0;
@@ -45,54 +45,21 @@ public class ExtensionJobList {
public ExtensionJobList(int numJobs) {
this.numJobs = numJobs;
- job = new ArrayList<JobElement>();
+ job = new ArrayList<>();
}
- public ExtensionJobList(int numJobs, List<JobElement> elements) {
+ public ExtensionJobList(int numJobs, List<String> extensionJobNames) {
this.numJobs = numJobs;
- this.job = elements;
- }
-
- public void addJob(JobElement element) {
- job.add(element);
+ this.job = extensionJobNames;
}
@Override
public String toString() {
- StringBuilder buffer = new StringBuilder();
- buffer.append(numJobs + "\n\n");
- for (JobElement element : job) {
- buffer.append(element.toString());
- }
- return buffer.toString();
- }
-
- /**
- * Element for a job.
- */
- public static class JobElement {
- @XmlElement
- public String jobName;
-
- @XmlElement
- public EntityList jobEntities;
-
- public JobElement() {
- jobName = null;
- jobEntities = null;
- }
-
- public JobElement(String name, EntityList entities) {
- jobName = name;
- jobEntities = entities;
- }
-
- @Override
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- buffer.append("Job: " + jobName + ", #. entities: ");
- buffer.append(jobEntities.toString() + "\n");
- return buffer.toString();
+ StringBuilder builder = new StringBuilder();
+ builder.append(numJobs).append("\n");
+ for (String extensionJobNames : job) {
+ builder.append(extensionJobNames);
}
+ return builder.toString();
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
index b6ac79d..acb5cf4 100644
--- a/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
@@ -45,7 +45,7 @@ import java.util.List;
@NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSION_JOBS, query = "select OBJECT(a) from ExtensionJobsBean a "),
@NamedQuery(name = PersistenceConstants.DELETE_EXTENSION_JOB, query = "delete from ExtensionJobsBean a where a.jobName = :jobName "),
@NamedQuery(name = PersistenceConstants.GET_EXTENSION_JOB, query = "select OBJECT(a) from ExtensionJobsBean a where a.jobName = :jobName"),
- @NamedQuery(name = PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION, query = "select OBJECT(a) from ExtensionJobsBean a where a.extensionName = :extensionName")
+ @NamedQuery(name = PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION, query = "select a.jobName from ExtensionJobsBean a where a.extensionName = :extensionName")
})
//RESUME CHECKSTYLE CHECK LineLengthCheck
public class ExtensionJobsBean {
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/docs/src/site/twiki/falconcli/ExtensionList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/falconcli/ExtensionList.twiki b/docs/src/site/twiki/falconcli/ExtensionList.twiki
index 8ddf573..e55f88b 100644
--- a/docs/src/site/twiki/falconcli/ExtensionList.twiki
+++ b/docs/src/site/twiki/falconcli/ExtensionList.twiki
@@ -8,9 +8,9 @@ Usage:
$FALCON_HOME/bin/falcon extension -list -extensionName <<extension-name>>
Optional Args : -doAs <<user-name>>
--sortOrder <<sortOrder>> -offset <<offset-number>> -numResults <<number-of-results>> -fields <<field1,field2>>
+-sortOrder <<sortOrder>>
<a href="../restapi/ExtensionList.html">Parameters and examples described here.</a>
Example:
-$FALCON_HOME/bin/falcon extension -list -extensionName hdfs-mirroring -offset 0 -numResults 10
+$FALCON_HOME/bin/falcon extension -list -extensionName hdfs-mirroring
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/docs/src/site/twiki/restapi/ExtensionList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/ExtensionList.twiki b/docs/src/site/twiki/restapi/ExtensionList.twiki
index dcd4d8d..0a43a80 100644
--- a/docs/src/site/twiki/restapi/ExtensionList.twiki
+++ b/docs/src/site/twiki/restapi/ExtensionList.twiki
@@ -10,18 +10,15 @@ List jobs generated from an extension.
---++ Parameters
* :extension-name Name of the extension.
* sortOrder <optional> Sort order by job name. Valid options: "asc" (default) and "desc".
- * offset <optional> Show results from the offset. Default is 0.
- * numResults <optional> Number of results to show per request. Default is 10.
- * fields <optional> Output fields separated by commas. Valid options: STATUS, TAGS, PIPELINES, CLUSTERS.
* doAs <optional> Impersonate the user.
---++ Results
-Total number of results and a list of jobs generated from the extension, followed by the associated entities.
+Total number of results and a list of jobs implementing the given extension.
---++ Examples
---+++ Rest Call
<verbatim>
-GET http://localhost:15000/api/extensions/list/billCollection?fields=status,clusters,tags
+GET http://localhost:15000/api/extensions/list/billCollection
</verbatim>
---+++ Result
<verbatim>
@@ -29,42 +26,8 @@ GET http://localhost:15000/api/extensions/list/billCollection?fields=status,clus
"numJobs":"2",
"jobs":{
"job": [
- {
- "jobName": "daily-health-bill",
- "jobEntities: {
- "totalResults": "2",
- "entity": [
- {
- "type":"FEED",
- "name":"SampleUSHealthBill",
- "status":"SUBMITTED\u201d,
- "tags":{"tag":["related=ushealthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]},
- "clusters": {"cluster":["SampleCluster1","primaryCluster\u201d]}
- },
- {
- "type":"PROCESS\u201d,
- "name":"SampleBillPay\u201d,
- "status":"RUNNING\u201d,
- "tags":{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]},
- "clusters":{"cluster":"primaryCluster\u201d}
- }
- ]
- }
- },
- {
- "jobName": "fsa-bill",
- "jobEntities": {
- "totalResults": "1",
- "entity":
- {
- "type":"PROCESS\u201d,
- "name":"FSAPay\u201d,
- "status":"RUNNING\u201d,
- "tags\u201d:{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCollection","_falcon_extension_job=fsa-bill"]},
- "clusters":{"cluster":"primaryCluster\u201d}
- }
- }
- }
+ {"daily-health-bill"},
+ {"fsa-bill"}
]
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
index b47766c..18c8540 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -27,6 +27,7 @@ import org.apache.falcon.service.FalconJPAService;
import javax.persistence.EntityManager;
import javax.persistence.Query;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -74,10 +75,7 @@ public class ExtensionMetaStore {
} finally {
commitAndCloseTransaction(entityManager);
}
- if (resultSize > 0){
- return true;
- }
- return false;
+ return resultSize > 0;
}
public Boolean checkIfExtensionJobExists(String jobName) {
@@ -91,10 +89,7 @@ public class ExtensionMetaStore {
} finally {
commitAndCloseTransaction(entityManager);
}
- if (resultSize > 0){
- return true;
- }
- return false;
+ return resultSize > 0;
}
public List<ExtensionBean> getAllExtensions() {
@@ -102,7 +97,7 @@ public class ExtensionMetaStore {
beginTransaction(entityManager);
Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS);
try {
- return (List<ExtensionBean>)q.getResultList();
+ return (List<ExtensionBean>) q.getResultList();
} finally {
commitAndCloseTransaction(entityManager);
}
@@ -113,7 +108,7 @@ public class ExtensionMetaStore {
beginTransaction(entityManager);
Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE);
q.setParameter(EXTENSION_TYPE, extensionType);
- try{
+ try {
q.executeUpdate();
} finally {
commitAndCloseTransaction(entityManager);
@@ -128,7 +123,7 @@ public class ExtensionMetaStore {
try {
List resultList = q.getResultList();
if (!resultList.isEmpty()) {
- return (ExtensionBean)resultList.get(0);
+ return (ExtensionBean) resultList.get(0);
} else {
return null;
}
@@ -137,24 +132,26 @@ public class ExtensionMetaStore {
}
}
- public List<ExtensionJobsBean> getJobsForAnExtension(String extensionName) {
+ public List<String> getJobsForAnExtension(String extensionName) {
EntityManager entityManager = getEntityManager();
beginTransaction(entityManager);
Query query = entityManager.createNamedQuery(PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION);
query.setParameter(EXTENSION_NAME, extensionName);
+ List<String> jobNames = new ArrayList<>();
try {
- return (List<ExtensionJobsBean>)query.getResultList();
+ jobNames.addAll((List<String>) query.getResultList());
} finally {
commitAndCloseTransaction(entityManager);
}
+ return jobNames;
}
- public void deleteExtension(String extensionName){
+ public void deleteExtension(String extensionName) {
EntityManager entityManager = getEntityManager();
beginTransaction(entityManager);
Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION);
q.setParameter(EXTENSION_NAME, extensionName);
- try{
+ try {
q.executeUpdate();
} finally {
commitAndCloseTransaction(entityManager);
@@ -165,7 +162,7 @@ public class ExtensionMetaStore {
byte[] config) {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
boolean alreadySubmitted = false;
- if (metaStore.getExtensionJobDetails(jobName) != null){
+ if (metaStore.getExtensionJobDetails(jobName) != null) {
alreadySubmitted = true;
}
ExtensionJobsBean extensionJobsBean = new ExtensionJobsBean();
@@ -195,7 +192,7 @@ public class ExtensionMetaStore {
beginTransaction(entityManager);
Query query = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION_JOB);
query.setParameter(JOB_NAME, jobName);
- try{
+ try {
query.executeUpdate();
} finally {
commitAndCloseTransaction(entityManager);
@@ -237,7 +234,7 @@ public class ExtensionMetaStore {
}
}
- public List<ExtensionJobsBean> getAllExtensionJobs() {
+ List<ExtensionJobsBean> getAllExtensionJobs() {
EntityManager entityManager = getEntityManager();
beginTransaction(entityManager);
Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSION_JOBS);
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index c3b4feb..c50d6de 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -28,7 +28,6 @@ import org.apache.falcon.extensions.ExtensionType;
import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.persistence.ExtensionBean;
-import org.apache.falcon.persistence.ExtensionJobsBean;
import org.apache.falcon.util.StartupProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -383,17 +382,6 @@ public final class ExtensionStore {
return (storePath != null);
}
- public List<String> getJobsForAnExtension(final String extensionName) throws FalconException {
- List<ExtensionJobsBean> extensionJobs = metaStore.getJobsForAnExtension(extensionName);
- List<String> extensionJobNames = new ArrayList<>();
- if (null != extensionJobs && !extensionJobs.isEmpty()) {
- for (ExtensionJobsBean extensionJobsBean : extensionJobs) {
- extensionJobNames.add(extensionJobsBean.getJobName());
- }
- }
- return extensionJobNames;
- }
-
public String updateExtensionStatus(final String extensionName, String currentUser, ExtensionStatus status) throws
FalconException {
validateStatusChange(extensionName, currentUser);
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
index e3327e8..4c49445 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -45,7 +45,7 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
private static ExtensionMetaStore stateStore;
@BeforeClass
- public void setup() throws Exception{
+ public void setup() throws Exception {
initExtensionStore();
this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
this.conf = dfsCluster.getConf();
@@ -58,7 +58,7 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
}
@Test
- public void testExtension(){
+ public void testExtension() {
//insert
stateStore.storeExtensionBean("test1", "test_location", ExtensionType.TRUSTED, "test_description",
"falconUser");
@@ -86,6 +86,8 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
//storing again to check for entity manager merge to let submission go forward.
stateStore.storeExtensionJob("job1", "test2", feeds, processes, config);
+ Assert.assertEquals(stateStore.getJobsForAnExtension("test2").size(), 1);
+ Assert.assertEquals(stateStore.getJobsForAnExtension("test2").get(0), "job1");
Assert.assertEquals(stateStore.getAllExtensionJobs().size(), 1);
Assert.assertEquals(stateStore.getExtensionJobDetails("job1").getFeeds().get(0), "testFeed");
stateStore.deleteExtensionJob("job1");
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
index 8ada576..c7da6e3 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
@@ -113,7 +113,7 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager {
}
private void canDeleteExtension(String extensionName) throws FalconException {
- ExtensionStore metaStore = ExtensionStore.get();
+ ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
List<String> extensionJobs = metaStore.getJobsForAnExtension(extensionName);
if (!extensionJobs.isEmpty()) {
LOG.error("Extension:{} cannot be unregistered as {} are instances of the extension", extensionName,
@@ -190,7 +190,7 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager {
private JSONObject buildExtensionDetailResult(final String extensionName) throws FalconException {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
- if (!metaStore.checkIfExtensionExists(extensionName)){
+ if (!metaStore.checkIfExtensionExists(extensionName)) {
throw new ValidationException("No extension resources found for " + extensionName);
}
@@ -247,12 +247,22 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager {
}
}
+ protected static void checkIfExtensionExists(String extensionName) {
+ ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
+ ExtensionBean extensionBean = metaStore.getDetail(extensionName);
+ if (extensionBean == null) {
+ LOG.error("Extension not found: " + extensionName);
+ throw FalconWebException.newAPIException("Extension not found:" + extensionName,
+ Response.Status.NOT_FOUND);
+ }
+ }
+
protected static void checkIfExtensionJobNameExists(String jobName, String extensionName) {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
ExtensionJobsBean extensionJobsBean = metaStore.getExtensionJobDetails(jobName);
if (extensionJobsBean != null && !extensionJobsBean.getExtensionName().equals(extensionName)) {
LOG.error("Extension job with name: " + extensionName + " already exists.");
- throw FalconWebException.newAPIException("Extension job with name: " + extensionName + " already exists.",
+ throw FalconWebException.newAPIException("Extension job with name: " + jobName + " already exists.",
Response.Status.INTERNAL_SERVER_ERROR);
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
index 1008884..efb5489 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
@@ -40,7 +40,6 @@ import org.apache.falcon.persistence.ExtensionJobsBean;
import org.apache.falcon.resource.InstancesResult;
import org.apache.falcon.resource.APIResult;
import org.apache.falcon.resource.AbstractExtensionManager;
-import org.apache.falcon.resource.EntityList;
import org.apache.falcon.resource.ExtensionInstanceList;
import org.apache.falcon.resource.ExtensionJobList;
import org.apache.falcon.security.CurrentUser;
@@ -65,7 +64,6 @@ import javax.ws.rs.core.Response;
import javax.xml.bind.JAXBException;
import java.util.Collections;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Arrays;
import java.util.Set;
@@ -86,7 +84,6 @@ import java.io.ByteArrayInputStream;
public class ExtensionManagerProxy extends AbstractExtensionManager {
public static final Logger LOG = LoggerFactory.getLogger(ExtensionManagerProxy.class);
- private static final String TAG_PREFIX_EXTENSION_NAME = "_falcon_extension_name=";
private static final String ASCENDING_SORT_ORDER = "asc";
private static final String DESCENDING_SORT_ORDER = "desc";
private Extension extension = new Extension();
@@ -97,31 +94,19 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
private EntityProxyUtil entityProxyUtil = new EntityProxyUtil();
private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json";
+
//SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
@GET
@Path("list/{extension-name}")
@Produces({MediaType.TEXT_XML, MediaType.APPLICATION_JSON})
public ExtensionJobList getExtensionJobs(
@PathParam("extension-name") String extensionName,
- @DefaultValue("") @QueryParam("fields") String fields,
@DefaultValue(ASCENDING_SORT_ORDER) @QueryParam("sortOrder") String sortOrder,
- @DefaultValue("0") @QueryParam("offset") Integer offset,
- @QueryParam("numResults") Integer resultsPerPage,
@DefaultValue("") @QueryParam("doAs") String doAsUser) {
checkIfExtensionServiceIsEnabled();
- resultsPerPage = resultsPerPage == null ? getDefaultResultsPerPage() : resultsPerPage;
+ checkIfExtensionExists(extensionName);
try {
- // get filtered entities
- List<Entity> entities = getEntityList("", "", "", TAG_PREFIX_EXTENSION_NAME + extensionName, "", doAsUser);
- if (entities.isEmpty()) {
- return new ExtensionJobList(0);
- }
-
- // group entities by extension job name
- Map<String, List<Entity>> groupedEntities = groupEntitiesByJob(entities);
-
- // sort by extension job name
- List<String> jobNames = new ArrayList<>(groupedEntities.keySet());
+ List<String> jobNames = ExtensionStore.getMetaStore().getJobsForAnExtension(extensionName);
switch (sortOrder.toLowerCase()) {
case DESCENDING_SORT_ORDER:
Collections.sort(jobNames, Collections.reverseOrder(String.CASE_INSENSITIVE_ORDER));
@@ -129,19 +114,8 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
default:
Collections.sort(jobNames, String.CASE_INSENSITIVE_ORDER);
}
-
- // pagination and format output
- int pageCount = getRequiredNumberOfResults(jobNames.size(), offset, resultsPerPage);
- HashSet<String> fieldSet = new HashSet<>(Arrays.asList(fields.toUpperCase().split(",")));
- ExtensionJobList jobList = new ExtensionJobList(pageCount);
- for (int i = offset; i < offset + pageCount; i++) {
- String jobName = jobNames.get(i);
- List<Entity> jobEntities = groupedEntities.get(jobName);
- EntityList entityList = new EntityList(buildEntityElements(fieldSet, jobEntities), jobEntities.size());
- jobList.addJob(new ExtensionJobList.JobElement(jobName, entityList));
- }
- return jobList;
- } catch (FalconException | IOException e) {
+ return new ExtensionJobList(jobNames.size(), jobNames);
+ } catch (Throwable e) {
LOG.error("Failed to get extension job list of " + extensionName + ": ", e);
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
@@ -783,18 +757,6 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
return entities;
}
- private Map<String, List<Entity>> groupEntitiesByJob(List<Entity> entities) {
- Map<String, List<Entity>> groupedEntities = new HashMap<>();
- for (Entity entity : entities) {
- String jobName = getJobNameFromTag(entity.getTags());
- if (!groupedEntities.containsKey(jobName)) {
- groupedEntities.put(jobName, new ArrayList<Entity>());
- }
- groupedEntities.get(jobName).add(entity);
- }
- return groupedEntities;
- }
-
private static void checkIfExtensionServiceIsEnabled() {
if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) {
LOG.error(ExtensionService.SERVICE_NAME + " is not enabled.");
@@ -802,6 +764,4 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
ExtensionService.SERVICE_NAME + " is not enabled.", Response.Status.NOT_FOUND);
}
}
-
-
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
index 349cfec..a4a207c 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
@@ -100,8 +100,6 @@ public class ExtensionManagerIT extends AbstractTestExtensionStore {
// validate job list results
ExtensionJobList jobs = context.getExtensionJobs("hdfs-mirroring", null, null, null, null, null);
Assert.assertEquals(jobs.numJobs, 2);
- Assert.assertEquals(jobs.job.get(0).jobName, JOB_NAME_1);
- Assert.assertEquals(jobs.job.get(1).jobName, JOB_NAME_2);
// list extension job instances
System.out.println("extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags");
http://git-wip-us.apache.org/repos/asf/falcon/blob/c6755685/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
index fd4d364..48df7fb 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
@@ -294,9 +294,6 @@ public class TestContext extends AbstractTestContext {
String resultsPerPage, String doAsUser) {
WebResource resource = this.service.path("api/extension/list/" + extensionName);
resource = addQueryParam(resource, "doAs", doAsUser);
- resource = addQueryParam(resource, "fields", fields);
- resource = addQueryParam(resource, "sortOrder", sortOrder);
- resource = addQueryParam(resource, "offset", offset);
resource = addQueryParam(resource, "numResults", resultsPerPage);
ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken)
.accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML)