You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by su...@apache.org on 2015/03/09 16:14:32 UTC
falcon git commit: FALCON-1020 validate command produces different
results when run via prism and server. Contributed by pavan kumar kolamuri
Repository: falcon
Updated Branches:
refs/heads/master c790761e0 -> 0c6f3911d
FALCON-1020 validate command produces different results when run via prism and server. Contributed by pavan kumar kolamuri
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/0c6f3911
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/0c6f3911
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/0c6f3911
Branch: refs/heads/master
Commit: 0c6f3911d2972f266f7e0aca003560b5d530ca7a
Parents: c790761
Author: Suhas Vasu <su...@inmobi.com>
Authored: Mon Mar 9 20:44:10 2015 +0530
Committer: Suhas Vasu <su...@inmobi.com>
Committed: Mon Mar 9 20:44:10 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 6 +
FALCON-1063.patch | 204 -------------------
client/src/main/resources/feed-0.1.xsd | 7 +-
.../apache/falcon/entity/FileSystemStorage.java | 2 +-
.../proxy/SchedulableEntityManagerProxy.java | 23 ++-
5 files changed, 34 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 453e44c..3b9b243 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -81,6 +81,9 @@ Trunk (Unreleased)
Seetharam)
OPTIMIZATIONS
+ FALCON-1063 Falcon CLI list entities operation throws NullPointerException
+ (Pallavi Rao via Suhas Vasu)
+
FALCON-987 params command inconsistent behavior
(pavan kumar kolamuri via Suhas Vasu)
@@ -98,6 +101,9 @@ Trunk (Unreleased)
(Suhas vasu)
BUG FIXES
+ FALCON-1020 validate command produces different results when run
+ via prism and server (pavan kumar kolamuri via Suhas Vasu)
+
FALCON-950 Rerun does not work on succeeded instances (Suhas Vasu)
FALCON-1048 Incorrect documentation for feed instacnce listing api. (Suhas
http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/FALCON-1063.patch
----------------------------------------------------------------------
diff --git a/FALCON-1063.patch b/FALCON-1063.patch
deleted file mode 100644
index 5aea017..0000000
--- a/FALCON-1063.patch
+++ /dev/null
@@ -1,204 +0,0 @@
-diff --git a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-index 5a4dce6..85d7263 100644
---- a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-+++ b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-@@ -116,11 +116,11 @@ public abstract class AbstractCleanupHandler {
- Entity entity) throws FalconException {
- try {
- final AccessControlList acl = entity.getACL();
-- if (acl == null) {
-- throw new FalconException("ACL for entity " + entity.getName() + " is empty");
-+ // To support backward compatibility, will only use the ACL owner only if present
-+ if (acl != null) {
-+ CurrentUser.authenticate(acl.getOwner()); // proxy user
- }
-
-- CurrentUser.authenticate(acl.getOwner()); // proxy user
- return HadoopClientFactory.get().createProxiedFileSystem(
- ClusterHelper.getConfiguration(cluster));
- } catch (Exception e) {
-diff --git a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-index b9fd37e..861f80f 100644
---- a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-+++ b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-@@ -19,11 +19,13 @@
- package org.apache.falcon.security;
-
- import org.apache.falcon.FalconException;
-+import org.apache.falcon.entity.v0.Entity;
- import org.apache.falcon.util.ReflectionUtils;
- import org.apache.falcon.util.StartupProperties;
- import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
- import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
-
-+import java.io.IOException;
- import java.net.InetAddress;
- import java.net.UnknownHostException;
-
-@@ -104,4 +106,16 @@ public final class SecurityUtil {
- "org.apache.falcon.security.DefaultAuthorizationProvider");
- return ReflectionUtils.getInstanceByClassName(providerClassName);
- }
-+
-+ public static void tryProxy(Entity entity) throws IOException, FalconException {
-+ if (entity != null && entity.getACL() != null && SecurityUtil.isAuthorizationEnabled()) {
-+ final String aclOwner = entity.getACL().getOwner();
-+ final String aclGroup = entity.getACL().getGroup();
-+
-+ if (SecurityUtil.getAuthorizationProvider().shouldProxy(
-+ CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) {
-+ CurrentUser.proxy(aclOwner, aclGroup);
-+ }
-+ }
-+ }
- }
-diff --git a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-index 8e2e544..0df59b2 100644
---- a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-+++ b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-@@ -55,6 +55,8 @@ public class LogCleanupServiceTest extends AbstractTestBase {
- + "sample2" + "/logs/job-2010-01-01-01-00/000");
- private final Path instanceLogPath4 = new Path("/projects/falcon/staging/falcon/workflows/process/"
- + "sample" + "/logs/latedata/2010-01-01-01-00");
-+ private final Path instanceLogPath5 = new Path("/projects/falcon/staging/falcon/workflows/process/"
-+ + "sample3" + "/logs/job-2010-01-01-01-00/000");
- private final Path feedInstanceLogPath = new Path("/projects/falcon/staging/falcon/workflows/feed/"
- + "impressionFeed" + "/logs/job-2010-01-01-01-00/testCluster/000");
- private final Path feedInstanceLogPath1 = new Path("/projects/falcon/staging/falcon/workflows/feed/"
-@@ -90,15 +92,22 @@ public class LogCleanupServiceTest extends AbstractTestBase {
- Process otherProcess = (Process) process.copy();
- otherProcess.setName("sample2");
- otherProcess.setFrequency(new Frequency("days(1)"));
-+ Process noACLProcess = (Process) process.copy();
-+ noACLProcess.setName("sample3");
-+ noACLProcess.setACL(null);
- ConfigurationStore.get().remove(EntityType.PROCESS,
- otherProcess.getName());
- ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);
-+ ConfigurationStore.get().remove(EntityType.PROCESS,
-+ noACLProcess.getName());
-+ ConfigurationStore.get().publish(EntityType.PROCESS, noACLProcess);
-
- fs.mkdirs(instanceLogPath);
- fs.mkdirs(instanceLogPath1);
- fs.mkdirs(instanceLogPath2);
- fs.mkdirs(instanceLogPath3);
- fs.mkdirs(instanceLogPath4);
-+ fs.mkdirs(instanceLogPath5);
-
- // fs.setTimes wont work on dirs
- fs.createNewFile(new Path(instanceLogPath, "oozie.log"));
-@@ -138,6 +147,7 @@ public class LogCleanupServiceTest extends AbstractTestBase {
- Assert.assertFalse(fs.exists(instanceLogPath));
- Assert.assertFalse(fs.exists(instanceLogPath1));
- Assert.assertFalse(fs.exists(instanceLogPath2));
-+ Assert.assertFalse(fs.exists(instanceLogPath5));
- Assert.assertTrue(fs.exists(instanceLogPath3));
- }
-
-diff --git a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-index a36d916..7f9b405 100644
---- a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-+++ b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-@@ -18,10 +18,17 @@
-
- package org.apache.falcon.security;
-
-+
-+import org.apache.falcon.FalconException;
-+import org.apache.falcon.entity.v0.process.*;
-+import org.apache.falcon.entity.v0.process.Process;
- import org.apache.falcon.util.StartupProperties;
-+import org.mockito.Mockito;
- import org.testng.Assert;
- import org.testng.annotations.Test;
-
-+import java.io.IOException;
-+
- /**
- * Unit test for Security utils.
- */
-@@ -81,4 +88,25 @@ public class SecurityUtilTest {
- Assert.assertEquals(SecurityUtil.getAuthorizationProvider().getClass(),
- DefaultAuthorizationProvider.class);
- }
-+
-+ @Test
-+ public void testTryProxy() throws IOException, FalconException {
-+ Process process = Mockito.mock(Process.class);
-+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
-+ final String currentUser = System.getProperty("user.name");
-+
-+ // When ACL not specified
-+ CurrentUser.authenticate(currentUser);
-+ SecurityUtil.tryProxy(process);
-+ Assert.assertEquals(CurrentUser.getUser(), currentUser);
-+
-+ ACL acl = new ACL();
-+ acl.setOwner("testuser");
-+ acl.setGroup("users");
-+ Mockito.when(process.getACL()).thenReturn(acl);
-+
-+ // When ACL is specified
-+ SecurityUtil.tryProxy(process);
-+ Assert.assertEquals(CurrentUser.getUser(), "testuser");
-+ }
- }
-diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-index 9a044d9..67a66c6 100644
---- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-@@ -406,23 +406,13 @@ public abstract class AbstractEntityManager {
- + "Can't be submitted again. Try removing before submitting.");
- }
-
-- tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
-+ SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
- validate(entity);
- configStore.publish(entityType, entity);
- LOG.info("Submit successful: ({}): {}", type, entity.getName());
- return entity;
- }
-
-- private void tryProxy(Entity entity) throws IOException, FalconException {
-- final String aclOwner = entity.getACL().getOwner();
-- final String aclGroup = entity.getACL().getGroup();
-- if (SecurityUtil.isAuthorizationEnabled()
-- && SecurityUtil.getAuthorizationProvider().shouldProxy(
-- CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) {
-- CurrentUser.proxy(aclOwner, aclGroup);
-- }
-- }
--
- /**
- * KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass.
- * existingEntity in config store will have teh decoration and equals check fails
-@@ -646,7 +636,7 @@ public abstract class AbstractEntityManager {
- // the user who requested list query has no permission to access this entity. Skip this entity
- continue;
- }
-- tryProxy(entity);
-+ SecurityUtil.tryProxy(entity);
-
- List<String> tags = EntityUtil.getTags(entity);
- List<String> pipelines = EntityUtil.getPipelines(entity);
-diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-index 6b022c9..15e94cd 100644
---- a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-@@ -141,14 +141,7 @@ public class FalconAuthorizationFilter implements Filter {
- try {
- EntityType type = EntityType.getEnum(entityType);
- Entity entity = EntityUtil.getEntity(type, entityName);
-- if (entity != null && entity.getACL() != null) {
-- final String aclOwner = entity.getACL().getOwner();
-- final String aclGroup = entity.getACL().getGroup();
-- if (authorizationProvider.shouldProxy(
-- authenticatedUGI, aclOwner, aclGroup)) {
-- CurrentUser.proxy(aclOwner, aclGroup);
-- }
-- }
-+ SecurityUtil.tryProxy(entity);
- } catch (FalconException ignore) {
- // do nothing
- }
http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/client/src/main/resources/feed-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/feed-0.1.xsd b/client/src/main/resources/feed-0.1.xsd
index 0168a35..a228b29 100644
--- a/client/src/main/resources/feed-0.1.xsd
+++ b/client/src/main/resources/feed-0.1.xsd
@@ -286,7 +286,7 @@
</xs:documentation>
</xs:annotation>
<xs:attribute type="location-type" name="type" use="required"/>
- <xs:attribute type="xs:string" name="path" use="required"/>
+ <xs:attribute type="non-empty-string" name="path" use="required"/>
</xs:complexType>
<xs:complexType name="partition">
<xs:attribute type="IDENTIFIER" name="name" use="required"/>
@@ -391,4 +391,9 @@
</xs:annotation>
<xs:attribute type="xs:string" name="uri" use="required"/>
</xs:complexType>
+ <xs:simpleType name="non-empty-string">
+ <xs:restriction base="xs:string">
+ <xs:minLength value="1"/>
+ </xs:restriction>
+ </xs:simpleType>
</xs:schema>
http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index fe93048..1ba7b9d 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -199,7 +199,7 @@ public class FileSystemStorage extends Configured implements Storage {
}
}
- if (locationForType == null) {
+ if (locationForType == null || StringUtils.isEmpty(locationForType.getPath())) {
return null;
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
index 5f711ee..7ba289a 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
@@ -163,8 +163,27 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana
@Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
@Override
- public APIResult validate(@Context HttpServletRequest request, @PathParam("type") String type) {
- return super.validate(request, type);
+ public APIResult validate(@Context final HttpServletRequest request, @PathParam("type") final String type) {
+ final HttpServletRequest bufferedRequest = getBufferedRequest(request);
+ EntityType entityType = EntityType.getEnum(type);
+ final Entity entity;
+ try {
+ entity = deserializeEntity(bufferedRequest, entityType);
+ bufferedRequest.getInputStream().reset();
+ } catch (Exception e) {
+ throw FalconWebException.newException("Unable to parse the request", Response.Status.BAD_REQUEST);
+ }
+ return new EntityProxy(type, entity.getName()) {
+ @Override
+ protected Set<String> getColosToApply() {
+ return getApplicableColos(type, entity);
+ }
+
+ @Override
+ protected APIResult doExecute(String colo) throws FalconException {
+ return getEntityManager(colo).invoke("validate", bufferedRequest, type);
+ }
+ }.execute();
}
@DELETE