You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/05/23 20:31:15 UTC
[2/2] git commit: FALCON-445 Propagate hive credentials defined in
cluster entity to hive-site.xml. Contributed by Sowmya Ramesh
FALCON-445 Propagate hive credentials defined in cluster entity to hive-site.xml. Contributed by Sowmya Ramesh
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3d3bde55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3d3bde55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3d3bde55
Branch: refs/heads/master
Commit: 3d3bde558027af1f6ceb914b396cf6a41682625e
Parents: b7afe36
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri May 23 11:31:03 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri May 23 11:31:03 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 5 ++++-
.../org/apache/falcon/entity/ClusterHelper.java | 20 ++++++++++++++++++++
.../falcon/workflow/OozieWorkflowBuilder.java | 6 ++++++
.../OozieProcessWorkflowBuilderTest.java | 3 ++-
.../resources/config/cluster/cluster-0.1.xml | 1 +
5 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e26c7d..e06fc80 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,7 +7,10 @@ Trunk (Unreleased)
NEW FEATURES
IMPROVEMENTS
- FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré
+ FALCON-445 Propagate hive credentials defined in cluster entity to
+ hive-site.xml (Sowmya Ramesh via Venkatesh Seetharam)
+
+ FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré
via Shwetha GS)
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
index 3bf9d95..5284d68 100644
--- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
@@ -27,6 +27,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
/**
* Helper to get end points relating to the cluster.
*/
@@ -135,4 +139,20 @@ public final class ClusterHelper {
}
return null;
}
+
+ public static Map<String, String> geHiveProperties(Cluster cluster) {
+ if (cluster.getProperties() != null) {
+ List<Property> properties = cluster.getProperties().getProperties();
+ if (properties != null && !properties.isEmpty()) {
+ Map<String, String> hiveProperties = new HashMap<String, String>();
+ for (Property prop : properties) {
+ if (prop.getName().startsWith("hive.")) {
+ hiveProperties.put(prop.getName(), prop.getValue());
+ }
+ }
+ return hiveProperties;
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
index f5ff27a..7d84938 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java
@@ -483,6 +483,12 @@ public abstract class OozieWorkflowBuilder<T extends Entity> extends WorkflowBui
"Registry interface is not defined in cluster: " + cluster.getName());
}
+ // Propagate the hive properties from cluster entity
+ Map<String, String> hiveProperties = ClusterHelper.geHiveProperties(cluster);
+ if (hiveProperties != null && !hiveProperties.isEmpty()) {
+ hiveCredentials.putAll(hiveProperties);
+ }
+
hiveCredentials.put(METASTOREURIS, metaStoreUrl);
hiveCredentials.put("hive.metastore.execute.setugi", "true");
hiveCredentials.put("hcatNode", metaStoreUrl.replace("thrift", "hcat"));
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
index 8cfa9fc..2522ca3 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
@@ -289,8 +289,9 @@ public class OozieProcessWorkflowBuilderTest extends AbstractTestBase {
props.put(prop.getName(), prop.getValue());
}
- // verify table props
+ // verify table and hive props
Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
+ expected.putAll(ClusterHelper.geHiveProperties(cluster));
for (Map.Entry<String, String> entry : props.entrySet()) {
if (expected.containsKey(entry.getKey())) {
Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/resources/config/cluster/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/process/src/test/resources/config/cluster/cluster-0.1.xml b/process/src/test/resources/config/cluster/cluster-0.1.xml
index 3780b3f..032cc77 100644
--- a/process/src/test/resources/config/cluster/cluster-0.1.xml
+++ b/process/src/test/resources/config/cluster/cluster-0.1.xml
@@ -39,5 +39,6 @@
<properties>
<property name="field1" value="value1"/>
<property name="field2" value="value2"/>
+ <property name="hive.metastore.client.socket.timeout" value="20"/>
</properties>
</cluster>