You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2017/01/19 05:20:30 UTC
falcon git commit: FALCON-2258 Improve error message for
definition/describe api in case extension does not exist
Repository: falcon
Updated Branches:
refs/heads/master ae510daa2 -> 6e162747b
FALCON-2258 Improve error message for definition/describe api in case extension does not exist
Author: sandeep <sa...@gmail.com>
Reviewers: @pallavi-rao
Closes #341 from sandeepSamudrala/FALCON-2258 and squashes the following commits:
6b10c9d [sandeep] Corrected error messages
b7fd5ab [sandeep] Incorporated review comments
9c99133 [sandeep] Incorporated review comments
80e533b [sandeep] FALCON-2258 Improve error message for definition/describe api in case extension does not exist
7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command
26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon
b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon
194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon
e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon
250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6e162747
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6e162747
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6e162747
Branch: refs/heads/master
Commit: 6e162747b090b6433d2e1813938d4f28f6926af3
Parents: ae510da
Author: sandeep <sa...@gmail.com>
Authored: Thu Jan 19 10:50:13 2017 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Jan 19 10:50:13 2017 +0530
----------------------------------------------------------------------
docs/src/site/twiki/FalconUserExtension.twiki | 0
.../extensions/jdbc/ExtensionMetaStore.java | 9 +++--
.../falcon/extensions/store/ExtensionStore.java | 38 ++++++++++++--------
.../resource/AbstractExtensionManager.java | 22 ++++++++----
.../resource/proxy/ExtensionManagerProxy.java | 4 +++
.../service/BacklogMetricEmitterService.java | 2 +-
6 files changed, 51 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/docs/src/site/twiki/FalconUserExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconUserExtension.twiki b/docs/src/site/twiki/FalconUserExtension.twiki
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
index 9126b67..df5d6c9 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -102,7 +102,7 @@ public class ExtensionMetaStore {
beginTransaction(entityManager);
Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS);
try {
- return q.getResultList();
+ return (List<ExtensionBean>)q.getResultList();
} finally {
commitAndCloseTransaction(entityManager);
}
@@ -126,7 +126,12 @@ public class ExtensionMetaStore {
Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION);
q.setParameter(EXTENSION_NAME, extensionName);
try {
- return (ExtensionBean)q.getSingleResult();
+ List resultList = q.getResultList();
+ if (!resultList.isEmpty()) {
+ return (ExtensionBean)resultList.get(0);
+ } else {
+ return null;
+ }
} finally {
commitAndCloseTransaction(entityManager);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 32b0cfd..c50d6de 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -75,7 +75,7 @@ public final class ExtensionStore {
private static final String RESOURCES_DIR = "resources";
private static final String LIBS_DIR = "libs";
- public static final String EXTENSION_STORE_URI = "extension.store.uri";
+ static final String EXTENSION_STORE_URI = "extension.store.uri";
private static final ExtensionStore STORE = new ExtensionStore();
@@ -103,10 +103,9 @@ public final class ExtensionStore {
ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extension)
? ExtensionType.TRUSTED : ExtensionType.CUSTOM;
String description = getShortDescription(extension);
- String recipeName = extension;
String location = storePath.toString() + '/' + extension;
String extensionOwner = System.getProperty("user.name");
- metaStore.storeExtensionBean(recipeName, location, extensionType, description, extensionOwner);
+ metaStore.storeExtensionBean(extension, location, extensionType, description, extensionOwner);
}
} catch (FalconException e) {
LOG.error("Exception in ExtensionMetaStore:", e);
@@ -143,17 +142,20 @@ public final class ExtensionStore {
}
}
- public Map<String, String> getExtensionArtifacts(final String extensionName) throws
- FalconException {
+ private Map<String, String> getExtensionArtifacts(final String extensionName) throws
+ FalconException {
Map<String, String> extensionFileMap = new HashMap<>();
Path extensionPath;
try {
RemoteIterator<LocatedFileStatus> fileStatusListIterator;
- if (AbstractExtension.isExtensionTrusted(extensionName)){
+ if (AbstractExtension.isExtensionTrusted(extensionName)) {
extensionPath = new Path(storePath, extensionName.toLowerCase());
fileStatusListIterator = fs.listFiles(extensionPath, true);
- }else{
+ } else {
ExtensionBean extensionBean = metaStore.getDetail(extensionName);
+ if (null == extensionBean) {
+ throw new StoreAccessException("Extension not found:" + extensionName);
+ }
extensionPath = new Path(extensionBean.getLocation());
FileSystem fileSystem = getHdfsFileSystem(extensionBean.getLocation());
fileStatusListIterator = fileSystem.listFiles(extensionPath, true);
@@ -175,7 +177,6 @@ public final class ExtensionStore {
}
-
public Map<String, String> getExtensionResources(final String extensionName) throws StoreAccessException {
Map<String, String> extensionFileMap = new HashMap<>();
try {
@@ -243,10 +244,10 @@ public final class ExtensionStore {
InputStream data;
ByteArrayOutputStream writer = new ByteArrayOutputStream();
- if (resourcePath.startsWith("file")){
+ if (resourcePath.startsWith("file")) {
data = fs.open(resourceFile);
IOUtils.copyBytes(data, writer, fs.getConf(), true);
- }else{
+ } else {
FileSystem fileSystem = getHdfsFileSystem(resourcePath);
data = fileSystem.open(resourceFile);
IOUtils.copyBytes(data, writer, fileSystem.getConf(), true);
@@ -257,7 +258,7 @@ public final class ExtensionStore {
}
}
- public List<String> getTrustedExtensions() throws StoreAccessException {
+ private List<String> getTrustedExtensions() throws StoreAccessException {
List<String> extensionList = new ArrayList<>();
try {
FileStatus[] fileStatuses = fs.listStatus(storePath);
@@ -297,17 +298,18 @@ public final class ExtensionStore {
throw new ValidationException(msg);
}
}
+
private FileSystem getHdfsFileSystem(String path) throws FalconException {
Configuration conf = new Configuration();
URI uri;
try {
uri = new URI(path);
- } catch (URISyntaxException e){
+ } catch (URISyntaxException e) {
LOG.error("Exception : ", e);
throw new FalconException(e);
}
conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority());
- return HadoopClientFactory.get().createFalconFileSystem(uri);
+ return HadoopClientFactory.get().createFalconFileSystem(uri);
}
@@ -362,7 +364,8 @@ public final class ExtensionStore {
LOG.info("Extension :" + extensionName + " registered successfully.");
return "Extension :" + extensionName + " registered successfully.";
}
- public String getResource(final String extensionName, final String resourceName) throws FalconException {
+
+ public String getResource(final String extensionName, final String resourceName) throws FalconException {
Map<String, String> resources = getExtensionArtifacts(extensionName);
if (resources.isEmpty()) {
throw new StoreAccessException("No extension resources found for " + extensionName);
@@ -382,7 +385,12 @@ public final class ExtensionStore {
public String updateExtensionStatus(final String extensionName, String currentUser, ExtensionStatus status) throws
FalconException {
validateStatusChange(extensionName, currentUser);
- if (metaStore.getDetail(extensionName).getStatus().equals(status)) {
+ ExtensionBean extensionBean = metaStore.getDetail(extensionName);
+ if (extensionBean == null) {
+ LOG.error("Extension not found: " + extensionName);
+ throw new FalconException("Extension not found:" + extensionName);
+ }
+ if (extensionBean.getStatus().equals(status)) {
throw new ValidationException(extensionName + " is already in " + status.toString() + " state.");
} else {
metaStore.updateExtensionStatus(extensionName, status);
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
index f7bf8e8..63974f2 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
@@ -180,13 +180,17 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager {
throw new ValidationException("No extension resources found for " + extensionName);
}
- ExtensionBean bean = metaStore.getDetail(extensionName);
+ ExtensionBean extensionBean = metaStore.getDetail(extensionName);
+ if (extensionBean == null) {
+ LOG.error("Extension not found: " + extensionName);
+ throw new FalconException("Extension not found:" + extensionName);
+ }
JSONObject resultObject = new JSONObject();
try {
- resultObject.put(NAME, bean.getExtensionName());
- resultObject.put(EXTENSION_TYPE, bean.getExtensionType());
- resultObject.put(EXTENSION_DESC, bean.getDescription());
- resultObject.put(EXTENSION_LOCATION, bean.getLocation());
+ resultObject.put(NAME, extensionBean.getExtensionName());
+ resultObject.put(EXTENSION_TYPE, extensionBean.getExtensionType());
+ resultObject.put(EXTENSION_DESC, extensionBean.getDescription());
+ resultObject.put(EXTENSION_LOCATION, extensionBean.getLocation());
} catch (JSONException e) {
LOG.error("Exception in buildDetailResults:", e);
throw new FalconException(e);
@@ -216,7 +220,13 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager {
protected static void checkIfExtensionIsEnabled(String extensionName) {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
- if (!metaStore.getDetail(extensionName).getStatus().equals(ExtensionStatus.ENABLED)) {
+ ExtensionBean extensionBean = metaStore.getDetail(extensionName);
+ if (extensionBean == null) {
+ LOG.error("Extension not found: " + extensionName);
+ throw FalconWebException.newAPIException("Extension not found:" + extensionName,
+ Response.Status.NOT_FOUND);
+ }
+ if (!extensionBean.getStatus().equals(ExtensionStatus.ENABLED)) {
LOG.error("Extension: " + extensionName + " is in disabled state.");
throw FalconWebException.newAPIException("Extension: " + extensionName + " is in disabled state.",
Response.Status.INTERNAL_SERVER_ERROR);
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
index 6c9087e..1008884 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
@@ -662,6 +662,8 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
validateExtensionName(extensionName);
try {
return new APIResult(APIResult.Status.SUCCEEDED, ExtensionStore.get().getResource(extensionName, README));
+ } catch (FalconException e) {
+ throw FalconWebException.newAPIException(e, Response.Status.BAD_REQUEST);
} catch (Throwable e) {
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
@@ -731,6 +733,8 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
try {
return new APIResult(APIResult.Status.SUCCEEDED, ExtensionStore.get().getResource(extensionName,
extensionName.toLowerCase() + EXTENSION_PROPERTY_JSON_SUFFIX));
+ } catch (FalconException e) {
+ throw FalconWebException.newAPIException(e, Response.Status.BAD_REQUEST);
} catch (Throwable e) {
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
index 5a323fd..2480c96 100644
--- a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
+++ b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
@@ -421,7 +421,7 @@ public final class BacklogMetricEmitterService implements FalconService,
if (status.getInstances().length > 0
&& status.getInstances()[0].status == InstancesResult.
WorkflowStatus.SUCCEEDED) {
- LOG.debug("Instance of nominaltime {} of entity {} has succeeded, removing "
+ LOG.debug("Instance of nominal time {} of entity {} has succeeded, removing "
+ "from backlog entries", nominalTimeStr, entity.getName());
backlogMetricStore.deleteMetricInstance(entity.getName(),
metricInfo.getCluster(), nominalTime, entity.getEntityType());