You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/12/15 12:30:25 UTC
falcon git commit: FALCON-2207 ACL check while deleting extensions
Repository: falcon
Updated Branches:
refs/heads/master 617d5ab94 -> d1052bf0a
FALCON-2207 ACL check while deleting extensions
Author: Pracheer Agarwal <pr...@inmobi.com>
Reviewers: @sandeepSamudrala, @pallavi-rao
Closes #322 from PracheerAgarwal/extension and squashes the following commits:
99af5f2 [Pracheer Agarwal] Merge branch 'master' of https://github.com/apache/falcon into extension
aca55dc [Pracheer Agarwal] FALCON-2207 indentation changes and review comments changes
e743d42 [Pracheer Agarwal] Merge branch 'master' of https://github.com/apache/falcon into extension
c5ddc4a [Pracheer Agarwal] FALCON-2207 backmerge
089094c [Pracheer Agarwal] FALCON-2207 ACL check while deleting extensions, code review changes
3149eef [Pracheer Agarwal] FALCON-2207 ACL check while deleting extensions, code review changes
7bb7b13 [Pracheer Agarwal] FALCON-2207 ACL check while deleting extensions
e300218 [Pracheer Agarwal] WIP
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/d1052bf0
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/d1052bf0
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/d1052bf0
Branch: refs/heads/master
Commit: d1052bf0a8a0ec13b859131b4dccec31d15cbbc6
Parents: 617d5ab
Author: Pracheer Agarwal <pr...@inmobi.com>
Authored: Thu Dec 15 18:00:06 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Dec 15 18:00:06 2016 +0530
----------------------------------------------------------------------
.../falcon/persistence/ExtensionBean.java | 15 +++-
.../falcon/persistence/ExtensionJobsBean.java | 1 -
.../org/apache/falcon/util/HdfsClassLoader.java | 2 +-
.../extensions/jdbc/ExtensionMetaStore.java | 3 +-
.../falcon/extensions/store/ExtensionStore.java | 50 +++++++------
.../extensions/jdbc/ExtensionMetaStoreTest.java | 6 +-
.../extensions/store/ExtensionStoreTest.java | 74 +++++++++++++-------
.../resource/extensions/ExtensionManager.java | 5 +-
8 files changed, 100 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/common/src/main/java/org/apache/falcon/persistence/ExtensionBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionBean.java
index 2cade5b..79cfe16 100644
--- a/common/src/main/java/org/apache/falcon/persistence/ExtensionBean.java
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionBean.java
@@ -74,6 +74,11 @@ public class ExtensionBean {
@Column(name = "creation_time")
private Date creationTime;
+ @Basic
+ @NotNull
+ @Column(name = "extension_owner")
+ private String extensionOwner;
+
public ExtensionType getExtensionType() {
return extensionType;
}
@@ -90,7 +95,6 @@ public class ExtensionBean {
this.creationTime = creationTime;
}
-
public String getExtensionName() {
return extensionName;
}
@@ -114,4 +118,13 @@ public class ExtensionBean {
public void setDescription(String description) {
this.description = description;
}
+
+ public String getExtensionOwner() {
+ return extensionOwner;
+ }
+
+ public void setExtensionOwner(String extensionOwner) {
+ this.extensionOwner = extensionOwner;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
index 2dc66f8..15a4dac 100644
--- a/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java
@@ -102,7 +102,6 @@ public class ExtensionJobsBean {
this.creationTime = creationTime;
}
-
public byte[] getConfig() {
return config;
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/common/src/main/java/org/apache/falcon/util/HdfsClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/util/HdfsClassLoader.java b/common/src/main/java/org/apache/falcon/util/HdfsClassLoader.java
index bacc092..23fc7f7 100644
--- a/common/src/main/java/org/apache/falcon/util/HdfsClassLoader.java
+++ b/common/src/main/java/org/apache/falcon/util/HdfsClassLoader.java
@@ -44,7 +44,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class HdfsClassLoader extends URLClassLoader {
private static final Logger LOG = LoggerFactory.getLogger(HdfsClassLoader.class);
- private static Map<String, HdfsClassLoader> classLoaderCache = new ConcurrentHashMap<String, HdfsClassLoader>();
+ private static Map<String, HdfsClassLoader> classLoaderCache = new ConcurrentHashMap<>();
private static final Object LOCK = new Object();
public static ClassLoader load(final String name, final List<String> jarHdfsPath) throws IOException {
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
index 4250e15..277cb95 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -42,13 +42,14 @@ public class ExtensionMetaStore {
}
public void storeExtensionBean(String extensionName, String location, ExtensionType extensionType,
- String description){
+ String description, String extensionOwner) {
ExtensionBean extensionBean = new ExtensionBean();
extensionBean.setLocation(location);
extensionBean.setExtensionName(extensionName);
extensionBean.setExtensionType(extensionType);
extensionBean.setCreationTime(new Date(System.currentTimeMillis()));
extensionBean.setDescription(description);
+ extensionBean.setExtensionOwner(extensionOwner);
EntityManager entityManager = getEntityManager();
try {
beginTransaction(entityManager);
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 76196b7..df63779 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -26,6 +26,7 @@ import org.apache.falcon.extensions.AbstractExtension;
import org.apache.falcon.extensions.ExtensionType;
import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.StartupProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -94,7 +95,7 @@ public final class ExtensionStore {
}
private void initializeDbTable() {
- try{
+ try {
metaStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
List<String> extensions = getExtensions();
for (String extension : extensions) {
@@ -103,9 +104,10 @@ public final class ExtensionStore {
String description = getShortDescription(extension);
String recipeName = extension;
String location = storePath.toString() + '/' + extension;
- metaStore.storeExtensionBean(recipeName, location, extensionType, description);
+ String extensionOwner = CurrentUser.getUser();
+ metaStore.storeExtensionBean(recipeName, location, extensionType, description, extensionOwner);
}
- } catch (FalconException e){
+ } catch (FalconException e) {
LOG.error("Exception in ExtensionMetaStore:", e);
throw new RuntimeException(e);
}
@@ -251,63 +253,65 @@ public final class ExtensionStore {
}
return extensionList;
}
- public String deleteExtension(final String extensionName) throws ValidationException{
+
+ public String deleteExtension(final String extensionName, String currentUser) throws FalconException {
ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED
: ExtensionType.CUSTOM;
- if (extensionType.equals(ExtensionType.TRUSTED)){
+ if (extensionType.equals(ExtensionType.TRUSTED)) {
throw new ValidationException(extensionName + " is trusted cannot be deleted.");
- }
- if (metaStore.checkIfExtensionExists(extensionName)) {
+ } else if (!metaStore.checkIfExtensionExists(extensionName)) {
+ throw new FalconException("Extension:" + extensionName + " is not registered with Falcon.");
+ } else if (!metaStore.getDetail(extensionName).getExtensionOwner().equals(currentUser)) {
+ throw new FalconException("User: " + currentUser + " is not allowed to delete extension: " + extensionName);
+ } else {
metaStore.deleteExtension(extensionName);
return "Deleted extension:" + extensionName;
- } else {
- return "Extension:" + extensionName + " is not registered with Falcon.";
}
}
- public String registerExtension(final String extensionName, final String path, final String description)
- throws URISyntaxException, FalconException {
+ public String registerExtension(final String extensionName, final String path, final String description,
+ String extensionOwner) throws URISyntaxException, FalconException {
Configuration conf = new Configuration();
URI uri = new URI(path);
conf.set("fs.defaultFS", uri.getScheme() + "://" + uri.getAuthority());
- FileSystem fileSystem = HadoopClientFactory.get().createFalconFileSystem(uri);
+ FileSystem fileSystem = HadoopClientFactory.get().createFalconFileSystem(uri);
try {
fileSystem.listStatus(new Path(uri.getPath() + "/README"));
- } catch (IOException e){
+ } catch (IOException e) {
LOG.error("Exception in registering Extension:{}", extensionName, e);
throw new ValidationException("README file is not present in the " + path);
}
- PathFilter filter=new PathFilter(){
- public boolean accept(Path file){
+ PathFilter filter = new PathFilter() {
+ public boolean accept(Path file) {
return file.getName().endsWith(".jar");
}
};
FileStatus[] jarStatus;
try {
jarStatus = fileSystem.listStatus(new Path(uri.getPath(), "libs/build"), filter);
- if (jarStatus.length <=0) {
+ if (jarStatus.length <= 0) {
throw new ValidationException("Jars are not present in the " + uri.getPath() + "/libs/build.");
}
- } catch (IOException e){
+ } catch (IOException e) {
LOG.error("Exception in registering Extension:{}", extensionName, e);
throw new ValidationException("Jars are not present in the " + uri.getPath() + "/libs/build.");
}
FileStatus[] propStatus;
- try{
+ try {
propStatus = fileSystem.listStatus(new Path(uri.getPath() + "/META"));
- if (propStatus.length <=0){
+ if (propStatus.length <= 0) {
throw new ValidationException("No properties file is not present in the " + uri.getPath() + "/META"
+ " structure.");
}
- } catch (IOException e){
+ } catch (IOException e) {
LOG.error("Exception in registering Extension:{}", extensionName, e);
throw new ValidationException("Directory is not present in the " + uri.getPath() + "/META"
+ " structure.");
}
- if (!metaStore.checkIfExtensionExists(extensionName)){
- metaStore.storeExtensionBean(extensionName, path, ExtensionType.CUSTOM, description);
- }else{
+ if (!metaStore.checkIfExtensionExists(extensionName)) {
+ metaStore.storeExtensionBean(extensionName, path, ExtensionType.CUSTOM, description, extensionOwner);
+ } else {
throw new ValidationException(extensionName + " already exists.");
}
return "Extension :" + extensionName + " registered successfully.";
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
index 099b5d1..1688abb 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -60,7 +60,8 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
@Test
public void testExtension(){
//insert
- stateStore.storeExtensionBean("test1", "test_location", ExtensionType.TRUSTED, "test_description");
+ stateStore.storeExtensionBean("test1", "test_location", ExtensionType.TRUSTED, "test_description",
+ "falconUser");
Assert.assertEquals(stateStore.getAllExtensions().size(), 1);
//check data
@@ -73,7 +74,8 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
@Test
public void testExtensionJob() {
- stateStore.storeExtensionBean("test2", "test_location", ExtensionType.CUSTOM, "test2_description");
+ stateStore.storeExtensionBean("test2", "test_location", ExtensionType.CUSTOM, "test2_description",
+ "falconUser");
List<String> processes = new ArrayList<>();
processes.add("testProcess");
List<String> feeds = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
index 50c9b7f..773fea2 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
@@ -44,7 +44,7 @@ import java.net.URISyntaxException;
import java.util.Map;
/**
- * Tests for extension store.
+ * Tests for extension store.
*/
public class ExtensionStoreTest extends AbstractTestExtensionStore {
private static Map<String, String> resourcesMap;
@@ -104,36 +104,61 @@ public class ExtensionStoreTest extends AbstractTestExtensionStore {
}
}
-
@Test
- public void testRegisterExtension() throws IOException, URISyntaxException, FalconException{
- createLibs();
- createReadmeAndJar();
- createMETA();
+ public void testRegisterExtension() throws IOException, URISyntaxException, FalconException {
+ String extensionPath = EXTENSION_PATH + "testRegister";
+ createLibs(extensionPath);
+ createReadmeAndJar(extensionPath);
+ createMETA(extensionPath);
store = ExtensionStore.get();
- store.registerExtension("test", STORAGE_URL + EXTENSION_PATH, "test desc");
+ store.registerExtension("test", STORAGE_URL + extensionPath, "test desc", "falconUser");
ExtensionMetaStore metaStore = new ExtensionMetaStore();
Assert.assertEquals(metaStore.getAllExtensions().size(), 1);
}
- @Test(expectedExceptions=ValidationException.class)
- public void testFailureCaseRegisterExtension() throws IOException, URISyntaxException, FalconException{
+ @Test(expectedExceptions = ValidationException.class)
+ public void testFailureCaseRegisterExtension() throws IOException, URISyntaxException, FalconException {
+ String extensionPath = EXTENSION_PATH + "testRegister";
+ store = ExtensionStore.get();
+ createLibs(extensionPath);
+ store.registerExtension("test", STORAGE_URL + EXTENSION_PATH, "test desc", "falconUser");
+ }
+
+ @Test
+ public void testDeleteExtension() throws IOException, URISyntaxException, FalconException {
+ String extensionPath = EXTENSION_PATH + "testDelete";
+ createLibs(extensionPath);
+ createReadmeAndJar(extensionPath);
+ createMETA(extensionPath);
+ store = ExtensionStore.get();
+ store.registerExtension("toBeDeleted", STORAGE_URL + extensionPath, "test desc", "falconUser");
+ store.deleteExtension("toBeDeleted", "falconUser");
+ ExtensionMetaStore metaStore = new ExtensionMetaStore();
+ Assert.assertEquals(metaStore.getAllExtensions().size(), 0);
+ }
+
+ @Test(expectedExceptions = FalconException.class)
+ public void testFailureDeleteExtension() throws IOException, URISyntaxException, FalconException {
+ String extensionPath = EXTENSION_PATH + "testACLOnDelete";
+ createLibs(extensionPath);
+ createReadmeAndJar(extensionPath);
+ createMETA(extensionPath);
store = ExtensionStore.get();
- createLibs();
- store.registerExtension("test", STORAGE_URL + EXTENSION_PATH, "test desc");
+ store.registerExtension("ACLFailure", STORAGE_URL + extensionPath, "test desc", "oozieUser");
+ store.deleteExtension("ACLFailure", "falconUser");
}
- private void createMETA() throws IOException{
- Path path = new Path(EXTENSION_PATH + "/META");
- if (fs.exists(path)){
+ private void createMETA(String extensionPath) throws IOException {
+ Path path = new Path(extensionPath + "/META");
+ if (fs.exists(path)) {
fs.delete(path, true);
}
fs.mkdirs(path);
- path = new Path(EXTENSION_PATH + "/META/test.properties");
+ path = new Path(extensionPath + "/META/test.properties");
OutputStream os = fs.create(path);
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
br.write("Hello World");
- if (fs.exists(path)){
+ if (fs.exists(path)) {
fs.delete(path, true);
}
br.write("test properties");
@@ -141,23 +166,23 @@ public class ExtensionStoreTest extends AbstractTestExtensionStore {
br.close();
}
- private void createLibs() throws IOException{
- Path path = new Path(EXTENSION_PATH);
- if (fs.exists(path)){
+ private void createLibs(String extensionPath) throws IOException {
+ Path path = new Path(extensionPath);
+ if (fs.exists(path)) {
fs.delete(path, true);
}
fs.mkdirs(path);
- path = new Path(EXTENSION_PATH + "/libs//libs/build");
+ path = new Path(extensionPath + "/libs//libs/build");
fs.mkdirs(path);
}
- private void createReadmeAndJar() throws IOException{
- Path path = new Path(EXTENSION_PATH + "/README");
- if (fs.exists(path)){
+ private void createReadmeAndJar(String extensionPath) throws IOException {
+ Path path = new Path(extensionPath + "/README");
+ if (fs.exists(path)) {
fs.delete(path, true);
}
fs.create(path);
- path = new Path(EXTENSION_PATH + "/libs/build/test.jar");
+ path = new Path(extensionPath + "/libs/build/test.jar");
OutputStream os = fs.create(path);
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
br.write("Hello World");
@@ -177,6 +202,5 @@ public class ExtensionStoreTest extends AbstractTestExtensionStore {
em.close();
}
}
-
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/d1052bf0/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
index 7c30c83..29859de 100644
--- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -46,6 +46,7 @@ import org.apache.falcon.resource.EntityList;
import org.apache.falcon.resource.ExtensionInstanceList;
import org.apache.falcon.resource.ExtensionJobList;
import org.apache.falcon.resource.InstancesResult;
+import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.service.Services;
import org.apache.falcon.util.DeploymentUtil;
import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -591,7 +592,7 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
checkIfExtensionServiceIsEnabled();
validateExtensionName(extensionName);
try {
- return ExtensionStore.get().deleteExtension(extensionName);
+ return ExtensionStore.get().deleteExtension(extensionName, CurrentUser.getUser());
} catch (Throwable e) {
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
@@ -608,7 +609,7 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
checkIfExtensionServiceIsEnabled();
validateExtensionName(extensionName);
try {
- return ExtensionStore.get().registerExtension(extensionName, path, description);
+ return ExtensionStore.get().registerExtension(extensionName, path, description, CurrentUser.getUser());
} catch (Throwable e) {
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}