You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sh...@apache.org on 2014/04/23 14:08:59 UTC
git commit: FALCON-380 The dependency option doesn't mention input or
output for a feed. Contributed by Suhas Vasu
Repository: incubator-falcon
Updated Branches:
refs/heads/master f10802d69 -> d28ad7e04
FALCON-380 The dependency option doesn't mention input or output for a feed. Contributed by Suhas Vasu
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d28ad7e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d28ad7e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d28ad7e0
Branch: refs/heads/master
Commit: d28ad7e04d9becbc180dc3dca5bf2e1186fdf4da
Parents: f10802d
Author: Shwetha GS <sh...@gmail.com>
Authored: Wed Apr 23 17:35:47 2014 +0530
Committer: Shwetha GS <sh...@gmail.com>
Committed: Wed Apr 23 17:38:40 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +
client/pom.xml | 5 ++
.../java/org/apache/falcon/cli/FalconCLI.java | 4 +-
.../org/apache/falcon/client/FalconClient.java | 18 ++---
.../org/apache/falcon/resource/EntityList.java | 78 ++++++++++++++++++--
.../site/twiki/restapi/EntityDependencies.twiki | 4 +-
.../falcon/resource/AbstractEntityManager.java | 4 +-
.../src/main/java/org/apache/falcon/Debug.java | 13 ++--
.../falcon/resource/EntityManagerJerseyIT.java | 12 +++
9 files changed, 113 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 51380f9..6f7589e 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -25,6 +25,9 @@ Trunk (Unreleased)
(Venkatesh Seetharam)
IMPROVEMENTS
+ FALCON-380 The dependency option doesn't mention input or output for a feed.
+ (Suhas Vasu via Shwetha GS)
+
FALCON-365 Remove the checked in oozie xsds. (Shwetha GS)
FALCON-356 Merge OozieProcessMapper and OozieProcessWorkflowBuilder. (Shwetha GS)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/client/pom.xml
----------------------------------------------------------------------
diff --git a/client/pom.xml b/client/pom.xml
index 63c4cce..7161f36 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -68,6 +68,11 @@
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
index 33f52c4..0c642dc 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -338,10 +338,10 @@ public class FalconCLI {
} else if (optionsList.contains(DEPENDENCY_OPT)) {
validateColo(optionsList);
validateEntityName(entityName);
- result = client.getDependency(entityType, entityName);
+ result = client.getDependency(entityType, entityName).toString();
} else if (optionsList.contains(LIST_OPT)) {
validateColo(optionsList);
- result = client.getEntityList(entityType);
+ result = client.getEntityList(entityType).toString();
} else if (optionsList.contains(HELP_CMD)) {
OUT.get().println("Falcon Help");
} else {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 5d9701f..59e3fd2 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -295,14 +295,12 @@ public class FalconClient {
entityName);
}
- public String getDependency(String entityType, String entityName)
+ public EntityList getDependency(String entityType, String entityName)
throws FalconCLIException {
-
- return sendDependencyRequest(Entities.DEPENDENCY, entityType,
- entityName);
+ return sendDependencyRequest(Entities.DEPENDENCY, entityType, entityName);
}
- public String getEntityList(String entityType) throws FalconCLIException {
+ public EntityList getEntityList(String entityType) throws FalconCLIException {
return sendListRequest(Entities.LIST, entityType);
}
@@ -487,7 +485,7 @@ public class FalconClient {
return clientResponse.getEntity(String.class);
}
- private String sendDependencyRequest(Entities entities, String entityType,
+ private EntityList sendDependencyRequest(Entities entities, String entityType,
String entityName) throws FalconCLIException {
ClientResponse clientResponse = service
@@ -501,7 +499,7 @@ public class FalconClient {
return parseEntityList(clientResponse);
}
- private String sendListRequest(Entities entities, String entityType)
+ private EntityList sendListRequest(Entities entities, String entityType)
throws FalconCLIException {
ClientResponse clientResponse = service
@@ -594,14 +592,14 @@ public class FalconClient {
return result.getMessage();
}
- private String parseEntityList(ClientResponse clientResponse)
+ private EntityList parseEntityList(ClientResponse clientResponse)
throws FalconCLIException {
EntityList result = clientResponse.getEntity(EntityList.class);
if (result == null || result.getElements() == null) {
- return "";
+ return null;
}
- return result.toString();
+ return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/client/src/main/java/org/apache/falcon/resource/EntityList.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/resource/EntityList.java b/client/src/main/java/org/apache/falcon/resource/EntityList.java
index 91ac00b..ae51172 100644
--- a/client/src/main/java/org/apache/falcon/resource/EntityList.java
+++ b/client/src/main/java/org/apache/falcon/resource/EntityList.java
@@ -18,12 +18,16 @@
package org.apache.falcon.resource;
+import org.apache.commons.lang.StringUtils;
import org.apache.falcon.entity.v0.Entity;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.process.Input;
+import org.apache.falcon.entity.v0.process.Output;
+import org.apache.falcon.entity.v0.process.Process;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.*;
+import java.util.ArrayList;
+import java.util.List;
/**
* Entity list used for marshalling / unmarshalling with REST calls.
@@ -47,11 +51,22 @@ public class EntityList {
public String name;
@XmlElement
public String status;
+ @XmlElementWrapper(name = "list")
+ public List<String> tag;
//RESUME CHECKSTYLE CHECK VisibilityModifierCheck
@Override
public String toString() {
- return "(" + type + ") " + name + "(" + status + ")\n";
+ String outString = "(" + type + ") " + name;
+ if (StringUtils.isNotEmpty(status)) {
+ outString += "(" + status + ")";
+ }
+
+ if (tag != null && !tag.isEmpty()) {
+ outString += " - " + tag.toString();
+ }
+ outString += "\n";
+ return outString;
}
}
@@ -72,7 +87,22 @@ public class EntityList {
EntityElement o = new EntityElement();
o.type = e.getEntityType().name().toLowerCase();
o.name = e.getName();
- o.status = "";
+ o.status = null;
+ items[i] = o;
+ }
+ this.elements = items;
+ }
+
+ public EntityList(Entity[] dependentEntities, Entity entity) {
+ int len = dependentEntities.length;
+ EntityElement[] items = new EntityElement[len];
+ for (int i = 0; i < len; i++) {
+ Entity e = dependentEntities[i];
+ EntityElement o = new EntityElement();
+ o.type = e.getEntityType().name().toLowerCase();
+ o.name = e.getName();
+ o.status = null;
+ o.tag = getEntityTag(e, entity);
items[i] = o;
}
this.elements = items;
@@ -86,8 +116,42 @@ public class EntityList {
public String toString() {
StringBuilder buffer = new StringBuilder();
for (EntityElement element : elements) {
- buffer.append(element);
+ buffer.append(element.toString());
}
return buffer.toString();
}
+
+ private List<String> getEntityTag(Entity dependentEntity, Entity entity) {
+ List<String> tagList = new ArrayList<String>();
+
+ if (entity.getEntityType().equals(EntityType.CLUSTER)) {
+ return tagList;
+ }
+
+ Process process = null;
+ String entityNameToMatch = null;
+ if (dependentEntity.getEntityType().equals(EntityType.PROCESS)) {
+ process = (Process) dependentEntity;
+ entityNameToMatch = entity.getName();
+ } else if (dependentEntity.getEntityType().equals(EntityType.FEED)
+ && entity.getEntityType().equals(EntityType.PROCESS)) {
+ process = (Process) entity;
+ entityNameToMatch = dependentEntity.getName();
+ }
+
+ if (process != null) {
+ for (Input i : process.getInputs().getInputs()) {
+ if (i.getFeed().equals(entityNameToMatch)) {
+ tagList.add("Input");
+ }
+ }
+ for (Output o : process.getOutputs().getOutputs()) {
+ if (o.getFeed().equals(entityNameToMatch)) {
+ tagList.add("Output");
+ }
+ }
+ }
+
+ return tagList;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/docs/src/site/twiki/restapi/EntityDependencies.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/EntityDependencies.twiki b/docs/src/site/twiki/restapi/EntityDependencies.twiki
index c61bc43..57fc256 100644
--- a/docs/src/site/twiki/restapi/EntityDependencies.twiki
+++ b/docs/src/site/twiki/restapi/EntityDependencies.twiki
@@ -25,11 +25,13 @@ GET http://localhost:15000/api/entities/dependencies/process/SampleProcess
"entity": [
{
"name": "SampleInput",
- "type": "feed"
+ "type": "feed",
+ "tag": [Input]
},
{
"name": "SampleOutput",
"type": "feed"
+ "tag": [Output]
},
{
"name": "primary-cluster",
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
index 2eeaf8a..e6f0709 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
@@ -438,8 +438,8 @@ public abstract class AbstractEntityManager {
try {
Entity entityObj = EntityUtil.getEntity(type, entityName);
Set<Entity> dependents = EntityGraph.get().getDependents(entityObj);
- Entity[] entities = dependents.toArray(new Entity[dependents.size()]);
- return new EntityList(entities);
+ Entity[] dependentEntities = dependents.toArray(new Entity[dependents.size()]);
+ return new EntityList(dependentEntities, entityObj);
} catch (Exception e) {
LOG.error("Unable to get dependencies for entityName " + entityName + "(" + type + ")", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/webapp/src/main/java/org/apache/falcon/Debug.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/Debug.java b/webapp/src/main/java/org/apache/falcon/Debug.java
index c831dc3..a650215 100644
--- a/webapp/src/main/java/org/apache/falcon/Debug.java
+++ b/webapp/src/main/java/org/apache/falcon/Debug.java
@@ -25,6 +25,8 @@ import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.process.Process;
+import org.apache.falcon.resource.EntityList;
+import org.apache.falcon.resource.EntityList.EntityElement;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.service.Services;
import org.apache.falcon.util.DeploymentProperties;
@@ -59,14 +61,13 @@ public final class Debug {
FalconClient client = new FalconClient(falconUrl);
for (int index = 2; index < args.length; index++) {
entity = args[index];
- String[] deps = client.getDependency(type, entity).split("\n");
- for (String line : deps) {
- String[] fields = line.replace("(", "").replace(")", "").split(" ");
- EntityType eType = EntityType.valueOf(fields[0].toUpperCase());
- if (ConfigurationStore.get().get(eType, fields[1]) != null) {
+ EntityList deps = client.getDependency(type, entity);
+ for (EntityElement dep : deps.getElements()) {
+ EntityType eType = EntityType.valueOf(dep.type.toUpperCase());
+ if (ConfigurationStore.get().get(eType, dep.name) != null) {
continue;
}
- String xml = client.getDefinition(eType.name().toLowerCase(), fields[1]);
+ String xml = client.getDefinition(eType.name().toLowerCase(), dep.name);
System.out.println(xml);
store(eType, xml);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d28ad7e0/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
index 4c0b733..8c27017 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
@@ -19,6 +19,7 @@ package org.apache.falcon.resource;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
+import org.apache.falcon.client.FalconClient;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.SchemaHelper;
@@ -31,6 +32,7 @@ import org.apache.falcon.entity.v0.process.Input;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.entity.v0.process.Property;
import org.apache.falcon.entity.v0.process.Validity;
+import org.apache.falcon.resource.EntityList.EntityElement;
import org.apache.falcon.util.BuildProperties;
import org.apache.falcon.util.DeploymentProperties;
import org.apache.falcon.util.OozieTestUtils;
@@ -227,6 +229,16 @@ public class EntityManagerJerseyIT {
context.scheduleProcess(tmpFile.getAbsolutePath(), overlay);
OozieTestUtils.waitForBundleStart(context, Status.FAILED, Status.KILLED);
+ FalconClient client = new FalconClient(TestContext.BASE_URL);
+ EntityList deps = client.getDependency(EntityType.PROCESS.name(), process.getName());
+ for (EntityElement dep : deps.getElements()) {
+ if (dep.name.equals(process.getInputs().getInputs().get(0).getName())) {
+ Assert.assertEquals("Input", dep.tag);
+ } else if (dep.name.equals(process.getOutputs().getOutputs().get(0).getName())) {
+ Assert.assertEquals("Output", dep.tag);
+ }
+ }
+
//Delete and re-submit the process with correct workflow
ClientResponse clientResponse = context.service
.path("api/entities/delete/process/" + context.processName)