You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2013/11/15 23:29:02 UTC
[1/3] git commit: FALCON-204 Process mapper generates params assuming
table input and output as names literally. Contributed by Venkatesh Seetharam
Updated Branches:
refs/heads/branch-0.4 624259368 -> c2731dcfd
FALCON-204 Process mapper generates params assuming table input and output as names literally. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/6a638c03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/6a638c03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/6a638c03
Branch: refs/heads/branch-0.4
Commit: 6a638c03fcc86308759ef81c9bf5deda5df15c8a
Parents: 6242593
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri Nov 15 14:20:19 2013 -0800
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri Nov 15 14:20:19 2013 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../apache/falcon/converter/OozieProcessMapper.java | 14 ++++++++++----
.../falcon/converter/OozieProcessMapperTest.java | 6 +++---
3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5cf9083..6bd84f3 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -178,6 +178,9 @@ Release Version: 0.4-incubating
FALCON-194 Import external libraries into the repository
+ FALCON-204 Process mapper generates params assuming table input and output
+ as names literally (Venkatesh Seetharam)
+
Release Version: 0.3-incubating
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
----------------------------------------------------------------------
diff --git a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
index eed7fa3..8749f07 100644
--- a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
+++ b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
@@ -375,9 +375,12 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper<Process> {
propagateCommonCatalogTableProperties(tableStorage, props, prefix);
- props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input', 'pig')}");
- props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input', 'hive')}");
- props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input', 'java')}");
+ props.put(prefix + "_partition_filter_pig",
+ "${coord:dataInPartitionFilter('" + input.getName() + "', 'pig')}");
+ props.put(prefix + "_partition_filter_hive",
+ "${coord:dataInPartitionFilter('" + input.getName() + "', 'hive')}");
+ props.put(prefix + "_partition_filter_java",
+ "${coord:dataInPartitionFilter('" + input.getName() + "', 'java')}");
}
private void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage,
@@ -386,7 +389,10 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper<Process> {
propagateCommonCatalogTableProperties(tableStorage, props, prefix);
- props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}");
+ props.put(prefix + "_dataout_partitions",
+ "${coord:dataOutPartitions('" + output.getName() + "')}");
+ props.put(prefix + "_dated_partition_value", "${coord:dataOutPartitionValue('"
+ + output.getName() + "', '" + tableStorage.getDatedPartitionKey() + "')}");
}
private String join(Iterator<String> itr, char sep) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
index b6f03e5..7d5f4d1 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
@@ -278,7 +278,7 @@ public class OozieProcessMapperTest extends AbstractTestBase {
Assert.assertNull(hiveAction.getPrepare());
Assert.assertEquals(Collections.EMPTY_LIST, hiveAction.getArchive());
Assert.assertFalse(hiveAction.getParam().isEmpty());
- Assert.assertEquals(10, hiveAction.getParam().size());
+ Assert.assertEquals(11, hiveAction.getParam().size());
}
@Test
@@ -356,11 +356,11 @@ public class OozieProcessMapperTest extends AbstractTestBase {
props.put(prefix + "_database", tableStorage.getDatabase());
props.put(prefix + "_table", tableStorage.getTable());
- if (prefix.equals("input")) {
+ if (prefix.equals("falcon_input")) {
props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input', 'pig')}");
props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input', 'hive')}");
props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input', 'java')}");
- } else if (prefix.equals("output")) {
+ } else if (prefix.equals("falcon_output")) {
props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}");
}
}
[2/3] git commit: FALCON-201 Package oozie-el-extensions jar files as
part of tarball. Contributed by Ashish Singh
Posted by ve...@apache.org.
FALCON-201 Package oozie-el-extensions jar files as part of tarball. Contributed by Ashish Singh
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/8be1cc12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/8be1cc12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/8be1cc12
Branch: refs/heads/branch-0.4
Commit: 8be1cc12587716efec8ff698be5b979119adc13e
Parents: 6a638c0
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri Nov 15 14:26:49 2013 -0800
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri Nov 15 14:26:49 2013 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +++
src/main/assemblies/distributed-package.xml | 5 +++++
src/main/assemblies/standalone-package.xml | 5 +++++
3 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8be1cc12/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6bd84f3..3d28b44 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -181,6 +181,9 @@ Release Version: 0.4-incubating
FALCON-204 Process mapper generates params assuming table input and output
as names literally (Venkatesh Seetharam)
+ FALCON-201 Package oozie-el-extensions jar files as part of tarball
+ (Ashish Singh via Venkatesh Seetharam)
+
Release Version: 0.3-incubating
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8be1cc12/src/main/assemblies/distributed-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
index 42704cb..0ca5017 100644
--- a/src/main/assemblies/distributed-package.xml
+++ b/src/main/assemblies/distributed-package.xml
@@ -107,6 +107,11 @@
<source>client/target/falcon-client-${project.version}.jar</source>
<outputDirectory>client/lib</outputDirectory>
</file>
+
+ <file>
+ <source>oozie-el-extensions/target/falcon-oozie-el-extension-${project.version}.jar</source>
+ <outputDirectory>oozie/libext</outputDirectory>
+ </file>
</files>
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8be1cc12/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index 1f307e9..8b35b0a 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -98,6 +98,11 @@
</file>
<file>
+ <source>oozie-el-extensions/target/falcon-oozie-el-extension-${project.version}.jar</source>
+ <outputDirectory>oozie/libext</outputDirectory>
+ </file>
+
+ <file>
<source>webapp/target/falcon-webapp-${project.version}.war</source>
<outputDirectory>server/webapp</outputDirectory>
<destName>falcon.war</destName>
[3/3] git commit: FALCON-203 Do not generate transitive closure for
the dependency graph. Contributed by Haohui Mai
Posted by ve...@apache.org.
FALCON-203 Do not generate transitive closure for the dependency graph. Contributed by Haohui Mai
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/c2731dcf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/c2731dcf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/c2731dcf
Branch: refs/heads/branch-0.4
Commit: c2731dcfda0f75a66e576c2a5fec415741e0ce4e
Parents: 8be1cc1
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri Nov 15 14:28:41 2013 -0800
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri Nov 15 14:28:41 2013 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +++
html5-ui/js/falcon-entity.js | 5 ++--
html5-ui/js/falcon.js | 55 +++++++++++++++++++--------------------
3 files changed, 33 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c2731dcf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3d28b44..f6cf839 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -184,6 +184,9 @@ Release Version: 0.4-incubating
FALCON-201 Package oozie-el-extensions jar files as part of tarball
(Ashish Singh via Venkatesh Seetharam)
+ FALCON-203 Do not generate transitive closure for the dependency graph
+ (Haohui Mai via Venkatesh Seetharam)
+
Release Version: 0.3-incubating
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c2731dcf/html5-ui/js/falcon-entity.js
----------------------------------------------------------------------
diff --git a/html5-ui/js/falcon-entity.js b/html5-ui/js/falcon-entity.js
index de2eae5..8ae1acb 100644
--- a/html5-ui/js/falcon-entity.js
+++ b/html5-ui/js/falcon-entity.js
@@ -111,6 +111,7 @@
var NODE_HEIGHT = 60;
var RECT_ROUND = 5;
var SEPARATION = 40;
+ var UNIVERSAL_SEP = 80;
// Function to draw the lines of the edge
var LINE_FUNCTION = d3.svg.line()
@@ -223,12 +224,12 @@
var n = nodes[key];
for (var i = 0, l = n.dependency.length; i < l; ++i) {
var d = n.dependency[i];
- g.addEdge(null, d, n.id);
+ g.addEdge(null, n.id, d);
}
}
layout = dagre.layout()
- .universalSep(100).rankSep(SEPARATION)
+ .universalSep(UNIVERSAL_SEP).rankSep(SEPARATION)
.run(g);
layout.eachEdge(drawEdge);
layout.eachNode(drawNode);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c2731dcf/html5-ui/js/falcon.js
----------------------------------------------------------------------
diff --git a/html5-ui/js/falcon.js b/html5-ui/js/falcon.js
index 14758bc..ff3a929 100644
--- a/html5-ui/js/falcon.js
+++ b/html5-ui/js/falcon.js
@@ -63,9 +63,6 @@
var nodes = {};
var next_node_id = 0;
- var STATE_UNINITIALIZED = 0;
- var STATE_REQUEST_SENT = 1;
-
var requests_in_fly = 0;
function key(type, name) {
@@ -78,7 +75,6 @@
return nodes[k];
var n = {
- "request_state": STATE_UNINITIALIZED,
"id": next_node_id++,
"type": type,
"name": name,
@@ -102,36 +98,39 @@
var l = data.entity.length;
for (var i = 0; i < l; ++i) {
var e = data.entity[i];
- node.dependency.push(getOrCreateNode(e.type, e.name).id);
- enqueue(e.type, e.name);
+ /**
+ * The REST API provides both the in and the out egeds
+ * of the dependency graph. Here we add the edeges based
+ * on the rules. (-> means the dependency edge)
+ *
+ * Feed->cluster, process->feed, process->cluster
+ */
+ var d = getOrCreateNode(e.type, e.name);
+ var src = null, dst = null;
+ if (d.type === "cluster") {
+ src = node; dst = d;
+ } else if (d.type === "process") {
+ src = d; dst = node;
+ } else {
+ if (node.type === "cluster") {
+ src = d; dst = node;
+ } else {
+ src = node; dst = d;
+ }
+ }
+ console.log(src.name + '->' + dst.name);
+ src.dependency.push(dst.id);
}
- })
- .always(function () {
- --requests_in_fly;
- if (requests_in_fly == 0)
- done_callback(nodes);
- });
+ done_callback(nodes);
+ })
}
- function enqueue(type, name) {
- // There's no dependency for clusters
- if (type === 'cluster')
- return;
-
- var k = key(type, name);
-
- if (nodes[k] !== undefined && nodes[k].request_state !== STATE_UNINITIALIZED)
- return;
-
- var n = getOrCreateNode(type, name);
- n.request_state = STATE_REQUEST_SENT;
-
- ++requests_in_fly;
+ function load() {
+ var n = getOrCreateNode(entity_type, entity_name);
loadEntry(n);
}
-
- enqueue(entity_type, entity_name);
+ load();
}
};