You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by je...@apache.org on 2015/02/25 23:56:01 UTC
phoenix git commit: PHOENIX-1078 Unable to run pig script with
Phoenix in a secure HBase cluster
Repository: phoenix
Updated Branches:
refs/heads/master 05723b19c -> dab9d51bf
PHOENIX-1078 Unable to run pig script with Phoenix in a secure HBase cluster
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dab9d51b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dab9d51b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dab9d51b
Branch: refs/heads/master
Commit: dab9d51bf5805e4f123e6095444d690f79250253
Parents: 05723b1
Author: Jeffrey Zhong <je...@apache.org>
Authored: Thu Feb 12 19:22:17 2015 -0800
Committer: Jeffrey Zhong <je...@apache.org>
Committed: Wed Feb 25 14:44:01 2015 -0800
----------------------------------------------------------------------
.../phoenix/mapreduce/util/ConnectionUtil.java | 9 +++++++++
.../mapreduce/util/PhoenixConfigurationUtil.java | 17 +++++++++++++++++
.../org/apache/phoenix/pig/PhoenixHBaseLoader.java | 9 ++++++---
.../apache/phoenix/pig/PhoenixHBaseStorage.java | 2 ++
4 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dab9d51b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
index 0864cba..364baf7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.mapreduce.util;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -42,6 +44,13 @@ public class ConnectionUtil {
public static Connection getConnection(final Configuration configuration) throws SQLException {
Preconditions.checkNotNull(configuration);
final Properties props = new Properties();
+ Iterator<Map.Entry<String, String>> iterator = configuration.iterator();
+ if(iterator != null) {
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+ props.setProperty(entry.getKey(), entry.getValue());
+ }
+ }
final Connection conn = DriverManager.getConnection(QueryUtil.getUrl(configuration.get(HConstants.ZOOKEEPER_QUORUM)), props);
return conn;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dab9d51b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index 83a606b..4d025ee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -19,13 +19,18 @@ package org.apache.phoenix.mapreduce.util;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -296,4 +301,16 @@ public final class PhoenixConfigurationUtil {
Preconditions.checkNotNull(configuration);
return configuration.get(OUTPUT_TABLE_NAME);
}
+
+ public static void loadHBaseConfiguration(Job job) throws IOException {
+ // load hbase-site.xml
+ Configuration hbaseConf = HBaseConfiguration.create();
+ for (Map.Entry<String, String> entry : hbaseConf) {
+ if (job.getConfiguration().get(entry.getKey()) == null) {
+ job.getConfiguration().set(entry.getKey(), entry.getValue());
+ }
+ }
+ //In order to have phoenix working on a secured cluster
+ TableMapReduceUtil.initCredentials(job);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dab9d51b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java
index 1218e82..18e362a 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java
@@ -106,12 +106,13 @@ public final class PhoenixHBaseLoader extends LoadFunc implements LoadMetadata {
}
@Override
- public void setLocation(String location, Job job) throws IOException {
+ public void setLocation(String location, Job job) throws IOException {
+ PhoenixConfigurationUtil.loadHBaseConfiguration(job);
+
final Configuration configuration = job.getConfiguration();
//explicitly turning off combining splits.
configuration.setBoolean("pig.noSplitCombination", true);
- //to have phoenix working on a secured cluster
- TableMapReduceUtil.initCredentials(job);
+
this.initializePhoenixPigConfiguration(location, configuration);
}
@@ -222,6 +223,8 @@ public final class PhoenixHBaseLoader extends LoadFunc implements LoadMetadata {
if(schema != null) {
return schema;
}
+
+ PhoenixConfigurationUtil.loadHBaseConfiguration(job);
final Configuration configuration = job.getConfiguration();
this.initializePhoenixPigConfiguration(location, configuration);
this.schema = PhoenixPigSchemaUtil.getResourceSchema(this.config);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dab9d51b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
index eb2c124..72d958b 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java
@@ -140,6 +140,8 @@ public class PhoenixHBaseStorage implements StoreFuncInterface {
if (!"hbase".equals(locationURI.getScheme())) {
throw new IOException(String.format("Location must use the hbase protocol, hbase://tableName[/columnList]. Supplied location=%s",location));
}
+
+ PhoenixConfigurationUtil.loadHBaseConfiguration(job);
config = job.getConfiguration();
config.set(HConstants.ZOOKEEPER_QUORUM, server);
String tableName = locationURI.getAuthority();