You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/11/21 11:44:35 UTC
falcon git commit: FALCON-2186 Rest api to get location of an
extension
Repository: falcon
Updated Branches:
refs/heads/master 03c4ca6fe -> 3d61e96fa
FALCON-2186 Rest api to get location of an extension
Dev testing :
bin/falcon extension -location -extensionName hdfs-mirroring
file:/usr/local/falcon/falcon-0.11-SNAPSHOT/extensions/hdfs-mirroring
Author: Praveen Adlakha <ad...@gmail.com>
Reviewers: @sandeepSamudrala, @pallavi-rao
Closes #301 from PraveenAdlakha/2186 and squashes the following commits:
de27a28 [Praveen Adlakha] comment's addressed
12ef6e9 [Praveen Adlakha] feature changed as per comment
ec02d96 [Praveen Adlakha] FALCON-2186 Rest api to get location of an extension
ead71da [Praveen Adlakha] comment's addressed
e256118 [Praveen Adlakha] checkstyle issues fixed
d5e27e5 [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe
f10707f [Praveen Adlakha] comments addressed
b9ee18f [Praveen Adlakha] test cases added and comments addressed
953cd4e [Praveen Adlakha] WIP
decd7e4 [Praveen Adlakha] sandeep's comment addressed
9964ffc [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3d61e96f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3d61e96f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3d61e96f
Branch: refs/heads/master
Commit: 3d61e96fa2076171f016529502f7c15dec2cb4a0
Parents: 03c4ca6
Author: Praveen Adlakha <ad...@gmail.com>
Authored: Mon Nov 21 17:14:23 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Mon Nov 21 17:14:23 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/falcon/cli/FalconCLI.java | 3 +-
.../apache/falcon/cli/FalconExtensionCLI.java | 9 ++++-
.../falcon/client/FalconCLIConstants.java | 1 +
.../org/apache/falcon/client/FalconClient.java | 9 ++++-
.../persistence/ExtensionMetadataBean.java | 1 -
.../persistence/PersistenceConstants.java | 1 -
.../extensions/jdbc/ExtensionMetaStore.java | 6 ++--
.../falcon/extensions/store/ExtensionStore.java | 1 -
.../extensions/jdbc/ExtensionMetaStoreTest.java | 4 ++-
.../resource/extensions/ExtensionManager.java | 36 ++++++++++++++++++++
10 files changed, 61 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
index 5b5f9ff..a071d7a 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -100,7 +100,8 @@ public class FalconCLI {
metadataCLI.createMetadataOptions(), true);
parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "",
"Extension operations like enumerate, definition, describe, list, instances, "
- + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister",
+ + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister"
+ + ",detail",
extensionCLI.createExtensionOptions(), true);
parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false);
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
----------------------------------------------------------------------
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
index 1bb27a0..4d023a6 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
@@ -50,6 +50,7 @@ public class FalconExtensionCLI {
public static final String DESCRIBE_OPT = "describe";
public static final String INSTANCES_OPT = "instances";
public static final String UNREGISTER_OPT = "unregister";
+ public static final String DETAIL_OPT = "detail";
// Input parameters
public static final String ENTENSION_NAME_OPT = "extensionName";
@@ -83,6 +84,9 @@ public class FalconExtensionCLI {
} else if (optionsList.contains(UNREGISTER_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
result = client.unregisterExtension(extensionName);
+ }else if (optionsList.contains(DETAIL_OPT)) {
+ validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
+ result = client.getExtensionDetail(extensionName);
} else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
@@ -157,7 +161,9 @@ public class FalconExtensionCLI {
Option suspend = new Option(FalconCLIConstants.SUSPEND_OPT, false, "Suspend an extension job");
Option resume = new Option(FalconCLIConstants.RESUME_OPT, false, "Resume an extension job");
Option delete = new Option(FalconCLIConstants.DELETE_OPT, false, "Delete an extension job");
- Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Delete metadata of extension job");
+ Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make"
+ + " the extension unavailable for instantiation");
+ Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension");
OptionGroup group = new OptionGroup();
group.addOption(enumerate);
@@ -174,6 +180,7 @@ public class FalconExtensionCLI {
group.addOption(resume);
group.addOption(delete);
group.addOption(unregister);
+ group.addOption(detail);
extensionOptions.addOptionGroup(group);
Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL");
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
index 8b15f86..3775771 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
@@ -218,4 +218,5 @@ public final class FalconCLIConstants {
public static final String DEBUG_OPTION_DESCRIPTION = "Use debug mode to see debugging statements on stdout";
public static final String DO_AS_DESCRIPTION = "doAs user";
public static final String UREGISTER = "unregister";
+ public static final String DETAIL = "detail";
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index ef9df7f..3db249a 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -347,7 +347,8 @@ public class FalconClient extends AbstractFalconClient {
SUSPEND("api/extension/suspend", HttpMethod.POST, MediaType.TEXT_XML),
RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML),
DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML),
- UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN);
+ UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN),
+ DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);;
private String path;
private String method;
@@ -1029,6 +1030,12 @@ public class FalconClient extends AbstractFalconClient {
return getResponse(String.class, clientResponse);
}
+ public String getExtensionDetail(final String extensionName) {
+ ClientResponse clientResponse = new ResourceBuilder().path(ExtensionOperations.DETAIL.path, extensionName)
+ .call(ExtensionOperations.DETAIL);
+ return getResponse(String.class, clientResponse);
+ }
+
public String getExtensionDefinition(final String extensionName) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.DEFINITION.path, extensionName)
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
index f4208ad..9f4cf72 100644
--- a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
@@ -37,7 +37,6 @@ import java.util.Date;
@Entity
@NamedQueries({
@NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "),
- @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"),
@NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType "),
@NamedQuery(name = PersistenceConstants.DELETE_EXTENSION, query = "delete from ExtensionMetadataBean a where a.extensionName = :extensionName "),
@NamedQuery(name = PersistenceConstants.GET_EXTENSION, query = "select OBJECT(a) from ExtensionMetadataBean a where a.extensionName = :extensionName")
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
index d0e8e9c..94eb32e 100644
--- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
+++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
@@ -68,7 +68,6 @@ public final class PersistenceConstants {
public static final String GET_ALL_BACKLOG_INSTANCES = "GET_ALL_BACKLOG_INSTANCES";
public static final String DELETE_ALL_BACKLOG_ENTITY_INSTANCES ="DELETE_ALL_BACKLOG_ENTITY_INSTANCES";
public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS";
- public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION";
public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE";
public static final String DELETE_EXTENSION = "DELETE_EXTENSION";
public static final String GET_EXTENSION = "GET_EXTENSION";
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
index 2c6003d..0a1a0e7 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -87,13 +87,13 @@ public class ExtensionMetaStore {
}
}
- public String getLocation(String extensionName){
+ public ExtensionMetadataBean getDetail(String extensionName){
EntityManager entityManager = getEntityManager();
beginTransaction(entityManager);
- Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION_LOCATION);
+ Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION);
q.setParameter("extensionName", extensionName);
try {
- return (String)q.getSingleResult();
+ return (ExtensionMetadataBean)q.getSingleResult();
} finally {
commitAndCloseTransaction(entityManager);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index ffb4341..02f7e62 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -250,7 +250,6 @@ public final class ExtensionStore {
}
return extesnionList;
}
-
public String deleteExtensionMetadata(final String extensionName) throws ValidationException{
ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED
: ExtensionType.CUSTOM;
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
index 7e0db0d..d0f1c0c 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -20,6 +20,7 @@ package org.apache.falcon.extensions.jdbc;
import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.falcon.extensions.ExtensionType;
import org.apache.falcon.extensions.store.AbstractTestExtensionStore;
+import org.apache.falcon.persistence.ExtensionMetadataBean;
import org.apache.falcon.service.FalconJPAService;
import org.apache.hadoop.conf.Configuration;
@@ -62,7 +63,8 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
Assert.assertEquals(stateStore.getAllExtensions().size(), 1);
//check data
- Assert.assertEquals(stateStore.getLocation("test1"), "test_location");
+ ExtensionMetadataBean bean = stateStore.getDetail("test1");
+ Assert.assertEquals(bean.getLocation(), "test_location");
//delete
stateStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
Assert.assertEquals(stateStore.getAllExtensions().size(), 0);
http://git-wip-us.apache.org/repos/asf/falcon/blob/3d61e96f/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
index 7fd4de5..b70b3a7 100644
--- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -21,6 +21,7 @@ package org.apache.falcon.resource.extensions;
import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.FalconWebException;
+import org.apache.falcon.entity.parser.ValidationException;
import org.apache.falcon.entity.store.StoreAccessException;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.cluster.Cluster;
@@ -420,6 +421,20 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
}
}
+ @GET
+ @Path("detail/{extension-name}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public Response getDetail(@PathParam("extension-name") String extensionName){
+ checkIfExtensionServiceIsEnabled();
+ validateExtensionName(extensionName);
+ try {
+ return Response.ok(buildDetailResult(extensionName)).build();
+ } catch (Throwable e) {
+ throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+
@POST
@Path("unregister/{extension-name}")
@Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@@ -524,6 +539,27 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
}
}
+ private JSONObject buildDetailResult(final String extensionName) throws FalconException {
+ ExtensionMetaStore metaStore = ExtensionStore.get().getMetaStore();
+
+ if (!metaStore.checkIfExtensionExists(extensionName)){
+ throw new ValidationException("No extension resources found for " + extensionName);
+ }
+
+ ExtensionMetadataBean bean = metaStore.getDetail(extensionName);
+ JSONObject resultObject = new JSONObject();
+ try {
+ resultObject.put(EXTENSION_NAME, bean.getExtensionName());
+ resultObject.put(EXTENSION_TYPE, bean.getExtensionType());
+ resultObject.put(EXTENSION_DESC, bean.getDescription());
+ resultObject.put(EXTENSION_LOCATION, bean.getLocation());
+ } catch (JSONException e) {
+ LOG.error("Exception in buildDetailResults:", e);
+ throw new FalconException(e);
+ }
+ return resultObject;
+ }
+
private Map<String, List<Entity>> groupEntitiesByJob(List<Entity> entities) {
Map<String, List<Entity>> groupedEntities = new HashMap<>();
for (Entity entity : entities) {