You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/11/17 10:20:14 UTC
falcon git commit: FALCON-2181 Support for storing metadata of non
trusted recipe
Repository: falcon
Updated Branches:
refs/heads/master df92d2b36 -> 2c9295a9d
FALCON-2181 Support for storing metadata of non trusted recipe
Author: Praveen Adlakha <ad...@gmail.com>
Reviewers: @pallavi-rao,@sandeepSamudrala
Closes #297 from PraveenAdlakha/751 and squashes the following commits:
c8428b7 [Praveen Adlakha] comments addressed
fb2c495 [Praveen Adlakha] test cases added and comments addressed
f59e622 [Praveen Adlakha] WIP
a69d899 [Praveen Adlakha] sandeep's comment addressed
d193679 [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/2c9295a9
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/2c9295a9
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/2c9295a9
Branch: refs/heads/master
Commit: 2c9295a9dac598336688f6389d158f6722d1686f
Parents: df92d2b
Author: Praveen Adlakha <ad...@gmail.com>
Authored: Thu Nov 17 15:49:57 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Nov 17 15:49:57 2016 +0530
----------------------------------------------------------------------
.../persistence/ExtensionMetadataBean.java | 112 +++++++++++++++
.../persistence/PersistenceConstants.java | 3 +
.../falcon/tools/FalconStateStoreDBCLI.java | 1 +
.../src/main/resources/META-INF/persistence.xml | 11 +-
extensions/pom.xml | 15 ++
.../apache/falcon/extensions/ExtensionType.java | 37 +++++
.../extensions/jdbc/ExtensionMetaStore.java | 101 ++++++++++++++
.../falcon/extensions/store/ExtensionStore.java | 50 ++++++-
.../falcon/extensions/ExtensionServiceTest.java | 4 +-
.../extensions/jdbc/ExtensionMetaStoreTest.java | 82 +++++++++++
.../store/AbstractTestExtensionStore.java | 35 +++++
.../test/resources/falcon-buildinfo.properties | 28 ++++
.../src/test/resources/startup.properties | 137 +++++++++++++++++++
.../resource/extensions/ExtensionManager.java | 37 ++---
src/build/findbugs-exclude.xml | 5 +
src/main/assemblies/standalone-package.xml | 9 --
.../falcon/resource/ExtensionManagerIT.java | 6 +
17 files changed, 633 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
new file mode 100644
index 0000000..ac1ded6
--- /dev/null
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.persistence;
+
+import javax.persistence.Basic;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.NamedQueries;
+import javax.persistence.NamedQuery;
+import javax.persistence.Table;
+import javax.validation.constraints.NotNull;
+import java.util.Date;
+
+
+//SUSPEND CHECKSTYLE CHECK LineLengthCheck
+/**
+ * Table to store extension metadata.
+ */
+
+@Table(name = "EXTENSION_METADATA")
+@Entity
+@NamedQueries({
+ @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "),
+ @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"),
+ @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType ")
+})
+//RESUME CHECKSTYLE CHECK LineLengthCheck
+public class ExtensionMetadataBean {
+ @Basic
+ @NotNull
+ @Id
+ @Column(name = "extension_name")
+ private String extensionName;
+
+
+ @Basic
+ @NotNull
+ @Column(name = "extension_type")
+ private String extensionType;
+
+ @Basic
+ @Column(name = "description")
+ private String description;
+
+ @Basic
+ @NotNull
+ @Column(name = "location")
+ private String location;
+
+
+ @Basic
+ @NotNull
+ @Column(name = "creation_time")
+ private Date creationTime;
+
+ public String getExtensionType() {
+ return extensionType;
+ }
+
+ public void setExtensionType(String extensionType) {
+ this.extensionType = extensionType;
+ }
+
+ public Date getCreationTime() {
+ return creationTime;
+ }
+
+ public void setCreationTime(Date creationTime) {
+ this.creationTime = creationTime;
+ }
+
+
+ public String getExtensionName() {
+ return extensionName;
+ }
+
+ public void setExtensionName(String extensionName) {
+ this.extensionName = extensionName;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ public void setLocation(String location) {
+ this.location = location;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
index 8be0eb5..dbbf06d 100644
--- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
+++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
@@ -67,4 +67,7 @@ public final class PersistenceConstants {
public static final String DELETE_BACKLOG_METRIC_INSTANCE = "DELETE_BACKLOG_METRIC_INSTANCE";
public static final String GET_ALL_BACKLOG_INSTANCES = "GET_ALL_BACKLOG_INSTANCES";
public static final String DELETE_ALL_BACKLOG_ENTITY_INSTANCES ="DELETE_ALL_BACKLOG_ENTITY_INSTANCES";
+ public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS";
+ public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION";
+ public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE";
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
index 9c6e8b3..e12b982 100644
--- a/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
+++ b/common/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
@@ -247,6 +247,7 @@ public class FalconStateStoreDBCLI {
args.add("org.apache.falcon.persistence.MonitoredEntityBean");
args.add("org.apache.falcon.persistence.EntitySLAAlertBean");
args.add("org.apache.falcon.persistence.BacklogMetricBean");
+ args.add("org.apache.falcon.persistence.ExtensionMetadataBean");
return args.toArray(new String[args.size()]);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/common/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/common/src/main/resources/META-INF/persistence.xml b/common/src/main/resources/META-INF/persistence.xml
index d58e21c..1fbcc9d 100644
--- a/common/src/main/resources/META-INF/persistence.xml
+++ b/common/src/main/resources/META-INF/persistence.xml
@@ -29,6 +29,7 @@
<class>org.apache.falcon.persistence.MonitoredEntityBean</class>
<class>org.apache.falcon.persistence.EntitySLAAlertBean</class>
<class>org.apache.falcon.persistence.BacklogMetricBean</class>
+ <class>org.apache.falcon.persistence.ExtensionMetadataBean</class>
<properties>
<property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
@@ -38,7 +39,8 @@
<property name="openjpa.MetaDataFactory"
value="jpa(Types=org.apache.falcon.persistence.EntityBean;
org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean;
- org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean)"></property>
+ org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean;
+ org.apache.falcon.persistence.ExtensionMetadataBean)"></property>
<property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
<property name="openjpa.LockManager" value="pessimistic"/>
@@ -62,6 +64,7 @@
<class>org.apache.falcon.persistence.MonitoredEntityBean</class>
<class>org.apache.falcon.persistence.EntitySLAAlertBean</class>
<class>org.apache.falcon.persistence.BacklogMetricBean</class>
+ <class>org.apache.falcon.persistence.ExtensionMetadataBean</class>
<properties>
<property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
@@ -70,7 +73,8 @@
<property name="openjpa.MetaDataFactory"
value="jpa(Types=org.apache.falcon.persistence.EntityBean;
org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean;
- org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean)"></property>
+ org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean;
+ org.apache.falcon.persistence.ExtensionMetadataBean)"></property>
<property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
<property name="openjpa.LockManager" value="pessimistic"/>
@@ -93,6 +97,7 @@
<class>org.apache.falcon.persistence.PendingInstanceBean</class>
<class>org.apache.falcon.persistence.EntitySLAAlertBean</class>
<class>org.apache.falcon.persistence.BacklogMetricBean</class>
+ <class>org.apache.falcon.persistence.ExtensionMetadataBean</class>
<properties>
<property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
@@ -102,7 +107,7 @@
value="jpa(Types=org.apache.falcon.persistence.EntityBean;
org.apache.falcon.persistence.InstanceBean;org.apache.falcon.persistence.PendingInstanceBean;
org.apache.falcon.persistence.MonitoredEntityBean;org.apache.falcon.persistence.EntitySLAAlertBean;
- org.apache.falcon.persistence.BacklogMetricBean)"/>
+ org.apache.falcon.persistence.BacklogMetricBean;org.apache.falcon.persistence.ExtensionMetadataBean)"/>
<property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
<property name="openjpa.LockManager" value="pessimistic"/>
<property name="openjpa.ReadLockLevel" value="read"/>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/pom.xml b/extensions/pom.xml
index 34fbad1..77821ca 100644
--- a/extensions/pom.xml
+++ b/extensions/pom.xml
@@ -103,6 +103,21 @@
<artifactId>falcon-test-util</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-dbcp</groupId>
+ <artifactId>commons-dbcp</artifactId>
+ <version>${commons-dbcp.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
new file mode 100644
index 0000000..c7e45cc
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+/**
+ * Enum to store ExtensionType.
+ */
+public enum ExtensionType {
+ TRUSTED("Trusted extension") ,
+ CUSTOM("Custom extension");
+
+ private final String text;
+
+ private ExtensionType(final String text) {
+ this.text = text;
+ }
+ @Override
+ public String toString(){
+ return text;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
new file mode 100644
index 0000000..b2f7e7d
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.extensions.jdbc;
+
+import org.apache.falcon.extensions.ExtensionType;
+import org.apache.falcon.persistence.ExtensionMetadataBean;
+import org.apache.falcon.persistence.PersistenceConstants;
+import org.apache.falcon.service.FalconJPAService;
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Statestore for extension framework.
+ */
+public class ExtensionMetaStore {
+
+ private EntityManager getEntityManager() {
+ return FalconJPAService.get().getEntityManager();
+ }
+
+ public void storeExtensionMetadataBean(String extensionName, String location, ExtensionType extensionType,
+ String description){
+ ExtensionMetadataBean extensionMetadataBean = new ExtensionMetadataBean();
+ extensionMetadataBean.setLocation(location);
+ extensionMetadataBean.setExtensionName(extensionName);
+ extensionMetadataBean.setExtensionType(extensionType.toString());
+ extensionMetadataBean.setCreationTime(new Date(System.currentTimeMillis()));
+ extensionMetadataBean.setDescription(description);
+ EntityManager entityManager = getEntityManager();
+ try {
+ beginTransaction(entityManager);
+ entityManager.persist(extensionMetadataBean);
+ } finally {
+ commitAndCloseTransaction(entityManager);
+ }
+ }
+
+ public List<ExtensionMetadataBean> getAllExtensions(){
+ EntityManager entityManager = getEntityManager();
+ beginTransaction(entityManager);
+ Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS);
+ try {
+ return q.getResultList();
+ } finally {
+ commitAndCloseTransaction(entityManager);
+ }
+ }
+
+ public void deleteExtensionsOfType(ExtensionType extensionType){
+ EntityManager entityManager = getEntityManager();
+ beginTransaction(entityManager);
+ Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE);
+ q.setParameter("extensionType", extensionType.toString());
+ try{
+ q.executeUpdate();
+ } finally {
+ commitAndCloseTransaction(entityManager);
+ }
+ }
+
+ public String getLocation(String extensionName){
+ EntityManager entityManager = getEntityManager();
+ beginTransaction(entityManager);
+ Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION_LOCATION);
+ q.setParameter("extensionName", extensionName);
+ try {
+ return (String)q.getSingleResult();
+ } finally {
+ commitAndCloseTransaction(entityManager);
+ }
+ }
+
+ private void beginTransaction(EntityManager entityManager) {
+ entityManager.getTransaction().begin();
+ }
+
+ private void commitAndCloseTransaction(EntityManager entityManager) {
+ if (entityManager != null) {
+ entityManager.getTransaction().commit();
+ entityManager.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 074cc73..61804f9 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -19,6 +19,10 @@
package org.apache.falcon.extensions.store;
import org.apache.commons.lang.StringUtils;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.extensions.AbstractExtension;
+import org.apache.falcon.extensions.ExtensionType;
+import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -39,6 +43,8 @@ import java.util.List;
import java.util.Map;
import java.util.HashMap;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,10 +54,19 @@ import org.slf4j.LoggerFactory;
public final class ExtensionStore {
private static final Logger LOG = LoggerFactory.getLogger(ExtensionStore.class);
+
+ public static ExtensionMetaStore getMetaStore() {
+ return metaStore;
+ }
+
+ private static ExtensionMetaStore metaStore = new ExtensionMetaStore();
private FileSystem fs;
private Path storePath;
+ private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json";
+ private static final String SHORT_DESCRIPTION = "shortDescription";
+
// Convention over configuration design paradigm
private static final String RESOURCES_DIR = "resources";
private static final String LIBS_DIR = "libs";
@@ -60,6 +75,7 @@ public final class ExtensionStore {
private static final ExtensionStore STORE = new ExtensionStore();
+
public static ExtensionStore get() {
return STORE;
}
@@ -72,6 +88,39 @@ public final class ExtensionStore {
}
storePath = new Path(uri);
fs = initializeFileSystem();
+ initializeDbTable();
+ }
+
+ private void initializeDbTable() {
+ try{
+ metaStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
+ List<String> extensions = getExtensions();
+ for (String extension : extensions) {
+ ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extension)
+ ? ExtensionType.TRUSTED : ExtensionType.CUSTOM;
+ String description = getShortDescription(extension);
+ String recipeName = extension;
+ String location = storePath.toString() + '/' + extension;
+ metaStore.storeExtensionMetadataBean(recipeName, location, extensionType, description);
+ }
+ } catch (FalconException e){
+ LOG.error("Exception in ExtensionStore:", e);
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private String getShortDescription(final String extensionName) throws FalconException {
+ String content = getResource(extensionName, extensionName.toLowerCase()
+ + EXTENSION_PROPERTY_JSON_SUFFIX);
+ String description;
+ try {
+ JSONObject jsonObject = new JSONObject(content);
+ description = (String) jsonObject.get(SHORT_DESCRIPTION);
+ } catch (JSONException e) {
+ throw new FalconException(e);
+ }
+ return description;
}
private FileSystem initializeFileSystem() {
@@ -83,7 +132,6 @@ public final class ExtensionStore {
// set permissions so config store dir is owned by falcon alone
HadoopClientFactory.mkdirs(fileSystem, storePath, HadoopClientFactory.ALL_PERMISSION);
}
-
return fileSystem;
} catch (Exception e) {
throw new RuntimeException("Unable to bring up extension store for path: " + storePath, e);
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java
index c8df2c0..a7c2fda 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionServiceTest.java
@@ -18,6 +18,7 @@
package org.apache.falcon.extensions;
+import org.apache.falcon.extensions.store.AbstractTestExtensionStore;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -26,12 +27,13 @@ import org.testng.Assert;
/**
* Unit tests for ExtensionService.
*/
-public class ExtensionServiceTest {
+public class ExtensionServiceTest extends AbstractTestExtensionStore {
private ExtensionService service;
@BeforeClass
public void setUp() throws Exception {
+ initExtensionStore();
service = new ExtensionService();
service.init();
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
new file mode 100644
index 0000000..61b26d6
--- /dev/null
+++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -0,0 +1,82 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.extensions.jdbc;
+
+import org.apache.falcon.cluster.util.EmbeddedCluster;
+import org.apache.falcon.extensions.ExtensionType;
+import org.apache.falcon.extensions.store.AbstractTestExtensionStore;
+import org.apache.falcon.service.FalconJPAService;
+
+import org.apache.hadoop.conf.Configuration;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+
+/**
+ * Test Cases for ExtensionMetaStore.
+ */
+
+
+public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
+ protected EmbeddedCluster dfsCluster;
+ protected Configuration conf = new Configuration();
+ private static ExtensionMetaStore stateStore;
+
+ @BeforeClass
+ public void setup() throws Exception{
+ initExtensionStore();
+ this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
+ this.conf = dfsCluster.getConf();
+ stateStore = new ExtensionMetaStore();
+ }
+
+ @BeforeMethod
+ public void init() {
+ clear();
+ }
+
+ @Test
+ public void dbOpertaions(){
+ //insert
+ stateStore.storeExtensionMetadataBean("test1", "test_location", ExtensionType.TRUSTED, "test_description");
+ Assert.assertEquals(stateStore.getAllExtensions().size(), 1);
+ //check data
+ Assert.assertEquals(stateStore.getLocation("test1"), "test_location");
+ //delete
+ stateStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
+ Assert.assertEquals(stateStore.getAllExtensions().size(), 0);
+ }
+
+ private void clear() {
+ EntityManager em = FalconJPAService.get().getEntityManager();
+ em.getTransaction().begin();
+ try {
+ Query query = em.createNativeQuery("delete from EXTENSION_METADATA");
+ query.executeUpdate();
+ } finally {
+ em.getTransaction().commit();
+ em.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
index fba803f..cd94eca 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
@@ -22,10 +22,15 @@ import org.apache.commons.io.FileUtils;
import org.apache.falcon.extensions.AbstractExtension;
import org.apache.falcon.extensions.ExtensionService;
import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.service.FalconJPAService;
+import org.apache.falcon.tools.FalconStateStoreDBCLI;
import org.apache.falcon.util.StartupProperties;
+import org.apache.falcon.util.StateStoreProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.testng.Assert;
import org.testng.annotations.AfterClass;
import java.io.File;
@@ -41,14 +46,44 @@ public class AbstractTestExtensionStore {
protected String extensionStorePath;
protected ExtensionStore store;
private FileSystem fileSystem;
+ protected LocalFileSystem fs = new LocalFileSystem();
+ private static final String DB_BASE_DIR = "target/test-data/persistancedb";
+ protected static final String DB_SQL_FILE = DB_BASE_DIR + File.separator + "out.sql";
+ protected static String dbLocation = DB_BASE_DIR + File.separator + "data.db";
+ protected static String url = "jdbc:derby:"+ dbLocation +";create=true";
+ private static FalconJPAService falconJPAService = FalconJPAService.get();
public void initExtensionStore() throws Exception {
initExtensionStore(this.getClass());
}
+ protected int execDBCLICommands(String[] args) {
+ return new FalconStateStoreDBCLI().run(args);
+ }
+
+ public void createDB(String file) {
+ File sqlFile = new File(file);
+ String[] argsCreate = { "create", "-sqlfile", sqlFile.getAbsolutePath(), "-run" };
+ int result = execDBCLICommands(argsCreate);
+ Assert.assertEquals(0, result);
+ Assert.assertTrue(sqlFile.exists());
+
+ }
+
public void initExtensionStore(Class resourceClass) throws Exception {
+ String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath();
+ String location = configPath + "-" + getClass().getName();
+ StartupProperties.get().setProperty("config.store.uri", location);
+ FileUtils.deleteDirectory(new File(location));
+ StateStoreProperties.get().setProperty(FalconJPAService.URL, url);
+ Configuration localConf = new Configuration();
+ fs.initialize(LocalFileSystem.getDefaultUri(localConf), localConf);
+ fs.mkdirs(new Path(DB_BASE_DIR));
+ createDB(DB_SQL_FILE);
+ falconJPAService.init();
new ExtensionService().init();
store = ExtensionService.getExtensionStore();
+ FalconJPAService falconJPAService = FalconJPAService.get();
fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true));
extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath();
extensionStoreSetup(resourceClass);
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/resources/falcon-buildinfo.properties
----------------------------------------------------------------------
diff --git a/extensions/src/test/resources/falcon-buildinfo.properties b/extensions/src/test/resources/falcon-buildinfo.properties
new file mode 100644
index 0000000..5a7cb82
--- /dev/null
+++ b/extensions/src/test/resources/falcon-buildinfo.properties
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+######################
+*.domain=all
+
+*.build.user=${user.name}
+*.build.epoch=${timestamp}
+*.project.version=${pom.version}
+*.build.version=${pom.version}-r${buildNumber}
+*.vc.revision=${buildNumber}
+*.vc.source.url=${scm.connection}
+######################
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/extensions/src/test/resources/startup.properties
----------------------------------------------------------------------
diff --git a/extensions/src/test/resources/startup.properties b/extensions/src/test/resources/startup.properties
new file mode 100644
index 0000000..58daf32
--- /dev/null
+++ b/extensions/src/test/resources/startup.properties
@@ -0,0 +1,137 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+*.domain=debug
+
+######### Implementation classes #########
+## DONT MODIFY UNLESS SURE ABOUT CHANGE ##
+
+*.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine
+*.oozie.process.workflow.builder=org.apache.falcon.workflow.OozieProcessWorkflowBuilder
+*.oozie.feed.workflow.builder=org.apache.falcon.workflow.OozieFeedWorkflowBuilder
+*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager
+*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService
+*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager
+*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService
+
+##### Falcon Services #####
+*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
+ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
+ org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.entity.store.ConfigurationStore,\
+ org.apache.falcon.rerun.service.RetryService,\
+ org.apache.falcon.rerun.service.LateRunService,\
+ org.apache.falcon.notification.service.impl.JobCompletionService,\
+ org.apache.falcon.notification.service.impl.SchedulerService,\
+ org.apache.falcon.notification.service.impl.AlarmService,\
+ org.apache.falcon.notification.service.impl.DataAvailabilityService,\
+ org.apache.falcon.execution.FalconExecutionService,\
+ org.apache.falcon.service.FalconJPAService
+
+##### Falcon Configuration Store Change listeners #####
+*.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap,\
+ org.apache.falcon.entity.store.FeedLocationStore
+
+##### JMS MQ Broker Implementation class #####
+*.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory
+
+
+######### System startup parameters #########
+
+# Location to store user entity configurations
+debug.config.store.uri=file://${user.dir}/target/store
+debug.config.store.persist=false
+debug.config.oozie.conf.uri=${user.dir}/target/oozie
+debug.system.lib.location=${system.lib.location}
+debug.broker.url=vm://localhost
+debug.retry.recorder.path=${user.dir}/target/retry
+debug.libext.feed.retention.paths=${falcon.libext}
+debug.libext.feed.replication.paths=${falcon.libext}
+debug.libext.process.paths=${falcon.libext}
+
+*.falcon.cleanup.service.frequency=minutes(5)
+
+*.extension.store.uri=file://${user.dir}/target/extensions
+
+
+######### Properties for configuring JMS provider - activemq #########
+# Default Active MQ url
+*.broker.url=tcp://localhost:61616
+
+# default time-to-live for a JMS message 3 days (time in minutes)
+*.broker.ttlInMins=4320
+*.entity.topic=FALCON.ENTITY.TOPIC
+*.max.retry.failure.count=1
+*.retry.recorder.path=${user.dir}/logs/retry
+
+######### Properties for configuring iMon client and metric #########
+*.internal.queue.size=1000
+
+
+##### List of shared libraries for Falcon workflows #####
+*.shared.libs=activemq-all,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3,commons-el
+
+######### Authentication Properties #########
+
+# Authentication type must be specified: simple|kerberos
+*.falcon.authentication.type=simple
+
+##### Service Configuration
+
+# Indicates the Kerberos principal to be used in Falcon Service.
+*.falcon.service.authentication.kerberos.principal=
+
+# Location of the keytab file with the credentials for the Service principal.
+*.falcon.service.authentication.kerberos.keytab=
+
+# name node principal to talk to config store
+*.dfs.namenode.kerberos.principal=
+
+##### SPNEGO Configuration
+
+# Authentication type must be specified: simple|kerberos|<class>
+# org.apache.falcon.security.RemoteUserInHeaderBasedAuthenticationHandler can be used for backwards compatibility
+*.falcon.http.authentication.type=simple
+
+# Indicates how long (in seconds) an authentication token is valid before it has to be renewed.
+*.falcon.http.authentication.token.validity=36000
+
+# The signature secret for signing the authentication tokens.
+*.falcon.http.authentication.signature.secret=falcon
+
+# The domain to use for the HTTP cookie that stores the authentication token.
+*.falcon.http.authentication.cookie.domain=
+
+# Indicates if anonymous requests are allowed when using 'simple' authentication.
+*.falcon.http.authentication.simple.anonymous.allowed=false
+
+# Indicates the Kerberos principal to be used for HTTP endpoint.
+# The principal MUST
+#rt with 'HTTP/' as per Kerberos HTTP SPNEGO specification.
+*.falcon.http.authentication.kerberos.principal=
+
+# Location of the keytab file with the credentials for the HTTP principal.
+*.falcon.http.authentication.kerberos.keytab=
+
+# The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's KerberosName for more details.
+*.falcon.http.authentication.kerberos.name.rules=DEFAULT
+
+# Comma separated list of black listed users
+*.falcon.http.authentication.blacklisted.users=
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
index 92b5531..138109c 100644
--- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -26,11 +26,12 @@ import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.process.Process;
-import org.apache.falcon.extensions.AbstractExtension;
import org.apache.falcon.extensions.Extension;
import org.apache.falcon.extensions.ExtensionProperties;
import org.apache.falcon.extensions.ExtensionService;
+import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.extensions.store.ExtensionStore;
+import org.apache.falcon.persistence.ExtensionMetadataBean;
import org.apache.falcon.resource.APIResult;
import org.apache.falcon.resource.AbstractSchedulableEntityManager;
import org.apache.falcon.resource.EntityList;
@@ -84,15 +85,12 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
private static final String EXTENSION_RESULTS = "extensions";
private static final String TOTAL_RESULTS = "totalResults";
private static final String README = "README";
- private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json";
- private static final String SHORT_DESCRIPTION = "shortDescription";
private static final String EXTENSION_NAME = "name";
private static final String EXTENSION_TYPE = "type";
private static final String EXTENSION_DESC = "description";
+ public static final String EXTENSION_LOCATION = "location";
- private static final String TRUSTED_EXTENSION = "Trusted extension";
- private static final String CUSTOM_EXTENSION = "Custom extension";
-
+ private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json";
//SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
@GET
@Path("list/{extension-name}")
@@ -446,17 +444,16 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
private static JSONArray buildEnumerateResult(final List<String> extensions) throws FalconException {
JSONArray results = new JSONArray();
-
- for (String extension : extensions) {
- String extensionType = AbstractExtension.isExtensionTrusted(extension) ? TRUSTED_EXTENSION
- : CUSTOM_EXTENSION;
-
+ ExtensionMetaStore metricStore = ExtensionStore.get().getMetaStore();
+ List<ExtensionMetadataBean> beanList = metricStore.getAllExtensions();
+ for (ExtensionMetadataBean bean : beanList) {
JSONObject resultObject = new JSONObject();
try {
- resultObject.put(EXTENSION_NAME, extension.toLowerCase());
- resultObject.put(EXTENSION_TYPE, extensionType);
- resultObject.put(EXTENSION_DESC, getShortDescription(extension));
+ resultObject.put(EXTENSION_NAME, bean.getExtensionName().toLowerCase());
+ resultObject.put(EXTENSION_TYPE, bean.getExtensionType());
+ resultObject.put(EXTENSION_DESC, bean.getDescription());
+ resultObject.put(EXTENSION_LOCATION, bean.getLocation());
} catch (JSONException e) {
throw new FalconException(e);
}
@@ -466,18 +463,6 @@ public class ExtensionManager extends AbstractSchedulableEntityManager {
return results;
}
- private static String getShortDescription(final String extensionName) throws FalconException {
- String content = ExtensionStore.get().getResource(extensionName, extensionName.toLowerCase()
- + EXTENSION_PROPERTY_JSON_SUFFIX);
- String description;
- try {
- JSONObject jsonObject = new JSONObject(content);
- description = (String) jsonObject.get(SHORT_DESCRIPTION);
- } catch (JSONException e) {
- throw new FalconException(e);
- }
- return description;
- }
private List<Entity> generateEntities(String extensionName, HttpServletRequest request)
throws FalconException, IOException {
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/src/build/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/src/build/findbugs-exclude.xml b/src/build/findbugs-exclude.xml
index 346583d..d910c96 100644
--- a/src/build/findbugs-exclude.xml
+++ b/src/build/findbugs-exclude.xml
@@ -70,6 +70,11 @@
</Match>
<Match>
+ <Class name="org.apache.falcon.persistence.ExtensionMetadataBean" />
+ <Bug pattern="NP_BOOLEAN_RETURN_NULL,UWF_UNWRITTEN_FIELD" />
+ </Match>
+
+ <Match>
<Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT" />
</Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index c3b2259..26df008 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -271,15 +271,6 @@
<directoryMode>0755</directoryMode>
</fileSet>
- <fileSet>
- <directory>../</directory>
- <outputDirectory>extensions/mirroring</outputDirectory>
- <excludes>
- <exclude>*/**</exclude>
- </excludes>
- <fileMode>0770</fileMode>
- <directoryMode>0770</directoryMode>
- </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/falcon/blob/2c9295a9/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
index db23d6d..349cfec 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
@@ -158,6 +158,12 @@ public class ExtensionManagerIT extends AbstractTestExtensionStore {
System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring");
Assert.assertEquals(TestContext.executeWithURL(
"extension -submitAndSchedule -extensionName hdfs-mirroring"), -1);
+
+ // enumerate
+
+ System.out.println("extension -enumerate");
+ Assert.assertEquals(TestContext.executeWithURL(
+ "extension -enumerate"), 3);
}
private Map<String, String> getHDFSMirroringProperty(String jobName, String start, String end) {