You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by so...@apache.org on 2016/08/04 22:04:01 UTC
falcon git commit: FALCON-2047 Use /apps/falcon/extensions/mirroring
as base dir for HiveDR instead of /apps/data-mirroring/
Repository: falcon
Updated Branches:
refs/heads/master 3efffd4d9 -> 723f1f7f1
FALCON-2047 Use /apps/falcon/extensions/mirroring as base dir for HiveDR instead of /apps/data-mirroring/
Author: Sowmya Ramesh <sr...@hortonworks.com>
Reviewers: "Venkat Ranganathan <ve...@hortonworks.com>, Balu Vellanki <ba...@apache.org>, Peeyush B <pb...@hortonworks.com>, Praveen Adlakha <ad...@gmail.com>"
Closes #252 from sowmyaramesh/FALCON-2047
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/723f1f7f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/723f1f7f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/723f1f7f
Branch: refs/heads/master
Commit: 723f1f7f1bb1cd052e81f757629f750393ae6a80
Parents: 3efffd4
Author: Sowmya Ramesh <so...@apache.org>
Authored: Thu Aug 4 15:03:57 2016 -0700
Committer: Sowmya Ramesh <sr...@hortonworks.com>
Committed: Thu Aug 4 15:03:57 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/falcon/hive/HiveDRTool.java | 17 +++++++++--------
.../org/apache/falcon/hive/util/DRStatusStore.java | 2 +-
.../org/apache/falcon/hive/util/EventUtils.java | 2 +-
.../falcon/extensions/store/ExtensionStore.java | 4 +---
.../src/test/resources/hive-mirroring-template.xml | 4 ++--
.../HdfsRecipe/hdfs-replication-template.xml | 2 +-
.../HdfsRecipe/hdfs-replication.properties | 2 +-
.../hive-disaster-recovery-template.xml | 2 +-
.../hive-disaster-recovery-secure-template.xml | 2 +-
falcon-ui/app/js/services/entity/entity-model.js | 4 ++--
src/main/assemblies/distributed-package.xml | 10 ++++++++++
src/main/assemblies/standalone-package.xml | 10 ++++++++++
.../src/test/resources/hive-mirroring-template.xml | 2 +-
13 files changed, 41 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
index 2701e02..7c6133d 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
@@ -136,13 +136,6 @@ public class HiveDRTool extends Configured implements Tool {
inputOptions = parseOptions(args);
LOG.info("Input Options: {}", inputOptions);
- // Update the source staging path
- inputOptions.setSourceStagingPath();
- inputOptions.setTargetStagingPath();
-
- LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
- LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
-
Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(),
inputOptions.getSourceNNKerberosPrincipal());
sourceClusterFS = FileSystem.get(sourceConf);
@@ -155,6 +148,14 @@ public class HiveDRTool extends Configured implements Tool {
// init DR status store
drStore = new HiveDRStatusStore(targetClusterFS);
+
+ // Update the source staging path after initing DR status store
+ inputOptions.setSourceStagingPath();
+ inputOptions.setTargetStagingPath();
+
+ LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
+ LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
+
eventSoucerUtil = new EventSourcerUtils(jobConf, inputOptions.shouldKeepHistory(), inputOptions.getJobName());
}
@@ -310,7 +311,7 @@ public class HiveDRTool extends Configured implements Tool {
}
private Map<String, Long> getLastDBTableEvents(Path lastEventIdFile) throws Exception {
- Map<String, Long> lastEventsIdMap = new HashMap<String, Long>();
+ Map<String, Long> lastEventsIdMap = new HashMap<>();
BufferedReader in = new BufferedReader(new InputStreamReader(jobFS.open(lastEventIdFile)));
try {
String line;
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
index cf6b7ad..86f50db 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
@@ -30,7 +30,7 @@ import java.util.List;
*/
public abstract class DRStatusStore {
- public static final String BASE_DEFAULT_STORE_PATH = "/apps/data-mirroring/";
+ public static final String BASE_DEFAULT_STORE_PATH = "/apps/falcon/extensions/mirroring";
public static final FsPermission DEFAULT_STORE_PERMISSION =
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.NONE);
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
index 492c70e..74d0eb6 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
@@ -50,7 +50,7 @@ import java.util.Map;
import java.util.Properties;
/**
- * Utility class to handle Hive events for data-mirroring.
+ * Utility class to handle Hive events for hive-mirroring.
*/
public class EventUtils {
private static final String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver";
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 4ca432a..b2cd7da 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -151,16 +151,14 @@ public final class ExtensionStore {
for (FileStatus fileStatus : files) {
if (fileStatus.getPath().getName().equalsIgnoreCase(LIBS_DIR)) {
- if (fileStatus.getLen() != 0) {
libsPath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
- }
break;
}
}
if (libsPath == null) {
LOG.info("For extension " + extensionName + " there is no "
- + LIBS_DIR + "at the extension store path " + storePath);
+ + LIBS_DIR + "at the extension store path " + extensionPath);
return null;
} else {
return libsPath.toString();
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/extensions/src/test/resources/hive-mirroring-template.xml
----------------------------------------------------------------------
diff --git a/extensions/src/test/resources/hive-mirroring-template.xml b/extensions/src/test/resources/hive-mirroring-template.xml
index 9f28991..fe0fae1 100644
--- a/extensions/src/test/resources/hive-mirroring-template.xml
+++ b/extensions/src/test/resources/hive-mirroring-template.xml
@@ -37,8 +37,8 @@
<property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
</properties>
- <workflow name="##workflow.name##" engine="oozie"
- path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+ <workflow name="##workflow.name##" engine="oozie" path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml"
+ lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
index 46546ce..b9834bb 100644
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
@@ -37,7 +37,7 @@
<property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
</properties>
- <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
+ <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
index 6c715f3..a3a1e33 100644
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
@@ -25,7 +25,7 @@ falcon.recipe.name=hdfs-replication-monthly
##### Workflow properties
falcon.recipe.workflow.name=hdfs-dr-workflow
# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
+falcon.recipe.workflow.path=/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml
##### Cluster properties
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
index c11c2bb..bff8313 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
+++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
@@ -38,7 +38,7 @@
</properties>
<workflow name="##workflow.name##" engine="oozie"
- path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+ path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
index c11c2bb..bff8313 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
+++ b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
@@ -38,7 +38,7 @@
</properties>
<workflow name="##workflow.name##" engine="oozie"
- path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+ path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-ui/app/js/services/entity/entity-model.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/entity/entity-model.js b/falcon-ui/app/js/services/entity/entity-model.js
index 48fa617..98d79c3 100644
--- a/falcon-ui/app/js/services/entity/entity-model.js
+++ b/falcon-ui/app/js/services/entity/entity-model.js
@@ -302,7 +302,7 @@
workflow: {
_name: "hdfs-dr-workflow",
_engine: "oozie",
- _path: "/apps/data-mirroring/workflows/hdfs-replication-workflow.xml",
+ _path: "/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml",
_lib: ""
},
retry: {
@@ -434,7 +434,7 @@
workflow: {
_name: "falcon-dr-hive-workflow",
_engine: "oozie",
- _path: "/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml",
+ _path: "/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml",
_lib: ""
},
retry: {
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/src/main/assemblies/distributed-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
index 4683a81..09665dc 100644
--- a/src/main/assemblies/distributed-package.xml
+++ b/src/main/assemblies/distributed-package.xml
@@ -248,6 +248,16 @@
</fileSet>
<fileSet>
+ <directory>../</directory>
+ <outputDirectory>extensions/mirroring</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <fileMode>0770</fileMode>
+ <directoryMode>0770</directoryMode>
+ </fileSet>
+
+ <fileSet>
<directory>../examples/app</directory>
<outputDirectory>examples/app</outputDirectory>
<directoryMode>0755</directoryMode>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index cfc3a26..c3b2259 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -270,6 +270,16 @@
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
+
+ <fileSet>
+ <directory>../</directory>
+ <outputDirectory>extensions/mirroring</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <fileMode>0770</fileMode>
+ <directoryMode>0770</directoryMode>
+ </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/webapp/src/test/resources/hive-mirroring-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hive-mirroring-template.xml b/webapp/src/test/resources/hive-mirroring-template.xml
index 9f28991..8d86408 100644
--- a/webapp/src/test/resources/hive-mirroring-template.xml
+++ b/webapp/src/test/resources/hive-mirroring-template.xml
@@ -38,7 +38,7 @@
</properties>
<workflow name="##workflow.name##" engine="oozie"
- path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+ path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>