You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pr...@apache.org on 2016/01/27 11:40:20 UTC
falcon git commit: FALCON-1664 Add Postgres support for native
scheduler.Contributed by Deepak Barr.
Repository: falcon
Updated Branches:
refs/heads/master 093ad2797 -> 5ff837819
FALCON-1664 Add Postgres support for native scheduler.Contributed by Deepak Barr.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/5ff83781
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/5ff83781
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/5ff83781
Branch: refs/heads/master
Commit: 5ff837819f5a6f31070d5f8bfbecacdd1eeb0ab9
Parents: 093ad27
Author: Pragya <mi...@gmail.com>
Authored: Wed Jan 27 10:31:35 2016 +0000
Committer: Pragya <mi...@gmail.com>
Committed: Wed Jan 27 10:31:35 2016 +0000
----------------------------------------------------------------------
CHANGES.txt | 3 +++
common/src/main/resources/statestore.properties | 2 +-
docs/src/site/twiki/FalconNativeScheduler.twiki | 2 +-
pom.xml | 1 +
scheduler/pom.xml | 10 ++++++++
.../falcon/state/store/jdbc/InstanceBean.java | 4 +--
.../falcon/tools/FalconStateStoreDBCLI.java | 3 ++-
.../src/main/resources/META-INF/persistence.xml | 27 ++++++++++++++++++++
src/conf/statestore.properties | 2 +-
9 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d87ce26..90b32ca 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,9 @@ Trunk
TASKS:
INCOMPATIBLE CHANGES
NEW FEATURES
+
+ FALCON-1664 Add Postgres support for native scheduler(Deepak Barr via Pragya Mittal)
+
FALCON-1495 In instance status list, show all runs for instances when requested by user(Narayan Periwal via Ajay Yadava)
FALCON-1230 Data based notification Service to notify execution instances when data becomes available(Pavan Kumar Kolamuri via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/common/src/main/resources/statestore.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/statestore.properties b/common/src/main/resources/statestore.properties
index a67a871..44e79b3 100644
--- a/common/src/main/resources/statestore.properties
+++ b/common/src/main/resources/statestore.properties
@@ -21,7 +21,7 @@
######### StateStore Properties #####
#*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore
#*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
-## Falcon currently supports derby and mysql, change url based on DB.
+## Falcon currently supports derby, mysql and postgreSQL, change url based on DB.
#*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true
## StateStore credentials file where username,password and other properties can be stored securely.
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/docs/src/site/twiki/FalconNativeScheduler.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconNativeScheduler.twiki b/docs/src/site/twiki/FalconNativeScheduler.twiki
index 6f0679e..de6bfe7 100644
--- a/docs/src/site/twiki/FalconNativeScheduler.twiki
+++ b/docs/src/site/twiki/FalconNativeScheduler.twiki
@@ -113,7 +113,7 @@ usage:
schema
</verbatim>
-Currently, MySQL and Derby are supported as state stores. We may extend support to other DBs in the future. Falcon has been tested against MySQL v5.5.
+Currently, MySQL, postgreSQL and Derby are supported as state stores. We may extend support to other DBs in the future. Falcon has been tested against MySQL v5.5 and PostgreSQL v9.5.
---++++ Using Derby as the State Store
Using Derby is ideal for QA and staging setup. Falcon comes bundled with a Derby connector and no explicit setup is required (although you can set it up) in terms creating the DB or tables.
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a80822f..94f7aee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@
<derby.version>10.10.1.1</derby.version>
<commons-dbcp.version>1.4</commons-dbcp.version>
<mysql.version>5.1.36</mysql.version>
+ <postgres.version>9.1-901-1.jdbc4</postgres.version>
<internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
<excluded.test.groups>exhaustive</excluded.test.groups>
</properties>
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/pom.xml
----------------------------------------------------------------------
diff --git a/scheduler/pom.xml b/scheduler/pom.xml
index 6392e98..2e2b2e9 100644
--- a/scheduler/pom.xml
+++ b/scheduler/pom.xml
@@ -145,6 +145,12 @@
</dependency>
<dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgres.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>${commons-dbcp.version}</version>
@@ -207,6 +213,10 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</artifactItem>
+ <artifactItem>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/dependency</outputDirectory>
</configuration>
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
index e8385b1..dffb116 100644
--- a/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
+++ b/scheduler/src/main/java/org/apache/falcon/state/store/jdbc/InstanceBean.java
@@ -114,11 +114,11 @@ public class InstanceBean {
private EntityBean entityBean;
- @Column(name = "awaited_predicates", columnDefinition = "BLOB")
+ @Column(name = "awaited_predicates")
@Lob
private byte[] awaitedPredicates;
- @Column(name = "properties", columnDefinition = "BLOB")
+ @Column(name = "properties")
@Lob
private byte[] properties;
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
index 7f22b0e..6de9f7d 100644
--- a/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
+++ b/scheduler/src/main/java/org/apache/falcon/tools/FalconStateStoreDBCLI.java
@@ -51,7 +51,8 @@ public class FalconStateStoreDBCLI {
// Represents whether DB instance exists or not.
private boolean instanceExists;
- private static final String[] FALCON_HELP = {"Falcon DB initialization tool currently supports Derby DB/ Mysql"};
+ private static final String[] FALCON_HELP =
+ {"Falcon DB initialization tool currently supports Derby DB/ Mysql/ PostgreSQL"};
public static void main(String[] args) {
new FalconStateStoreDBCLI().run(args);
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/scheduler/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/scheduler/src/main/resources/META-INF/persistence.xml b/scheduler/src/main/resources/META-INF/persistence.xml
index 99e116c..c2ef794 100644
--- a/scheduler/src/main/resources/META-INF/persistence.xml
+++ b/scheduler/src/main/resources/META-INF/persistence.xml
@@ -74,4 +74,31 @@
</properties>
</persistence-unit>
+ <persistence-unit name="falcon-postgresql" transaction-type="RESOURCE_LOCAL">
+ <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
+
+ <class>org.apache.falcon.state.store.jdbc.EntityBean</class>
+ <class>org.apache.falcon.state.store.jdbc.InstanceBean</class>
+
+ <properties>
+ <property name="openjpa.ConnectionDriverName" value="org.apache.commons.dbcp.BasicDataSource"/>
+
+ <property name="openjpa.ConnectionProperties" value="**INVALID**"/> <!--Set by StoreService at init time -->
+
+ <property name="openjpa.MetaDataFactory"
+ value="jpa(Types=org.apache.falcon.state.store.EntityBean;
+ org.apache.falcon.state.store.InstanceBean)"></property>
+
+ <property name="openjpa.DetachState" value="fetch-groups(DetachedStateField=true)"/>
+ <property name="openjpa.LockManager" value="pessimistic"/>
+ <property name="openjpa.ReadLockLevel" value="read"/>
+ <property name="openjpa.WriteLockLevel" value="write"/>
+ <property name="openjpa.jdbc.TransactionIsolation" value="repeatable-read"/> <!--CUSTOM-->
+ <property name="openjpa.jdbc.DBDictionary" value="batchLimit=50"/>
+ <property name="openjpa.jdbc.DBDictionary" value="TimestampTypeName=TIMESTAMP"/>
+ <property name="openjpa.RuntimeUnenhancedClasses" value="unsupported"/>
+ <property name="openjpa.Log" value="log4j"/>
+ </properties>
+ </persistence-unit>
+
</persistence>
http://git-wip-us.apache.org/repos/asf/falcon/blob/5ff83781/src/conf/statestore.properties
----------------------------------------------------------------------
diff --git a/src/conf/statestore.properties b/src/conf/statestore.properties
index de5ad8f..5659531 100644
--- a/src/conf/statestore.properties
+++ b/src/conf/statestore.properties
@@ -21,7 +21,7 @@
######### StateStore Properties #####
#*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore
#*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
-## Falcon currently supports derby and mysql, change url based on DB.
+## Falcon currently supports derby, mysql and postgreSQL, change url based on DB.
#*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true
## StateStore credentials file where username,password and other properties can be stored securely.