You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by je...@apache.org on 2014/10/16 23:23:24 UTC
git commit: PHOENIX-1353: CsvBulkLoad doesn't work in secure env
Repository: phoenix
Updated Branches:
refs/heads/4.0 6333e70af -> 4cd5826e7
PHOENIX-1353: CsvBulkLoad doesn't work in secure env
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4cd5826e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4cd5826e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4cd5826e
Branch: refs/heads/4.0
Commit: 4cd5826e7caf8e6db317f55ea974b077f29a3ece
Parents: 6333e70
Author: Jeffrey Zhong <je...@apache.org>
Authored: Thu Oct 16 14:13:14 2014 -0700
Committer: Jeffrey Zhong <je...@apache.org>
Committed: Thu Oct 16 14:16:28 2014 -0700
----------------------------------------------------------------------
.../end2end/ConnectionQueryServicesTestImpl.java | 2 +-
.../java/org/apache/phoenix/jdbc/PhoenixDriver.java | 2 +-
.../org/apache/phoenix/mapreduce/CsvBulkLoadTool.java | 6 +++++-
.../apache/phoenix/mapreduce/CsvToKeyValueMapper.java | 12 +++++++++++-
.../phoenix/query/ConnectionQueryServicesImpl.java | 9 ++++++++-
5 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cd5826e/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index 2020a43..bee8d21 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -36,7 +36,7 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
protected int NUM_SLAVES_BASE = 1; // number of slaves for the cluster
public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info) throws SQLException {
- super(services, info);
+ super(services, info, null);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cd5826e/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
index 10b15dc..6360d06 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
@@ -149,7 +149,7 @@ public final class PhoenixDriver extends PhoenixEmbeddedDriver {
if (normalizedConnInfo.isConnectionless()) {
connectionQueryServices = new ConnectionlessQueryServicesImpl(services, normalizedConnInfo);
} else {
- connectionQueryServices = new ConnectionQueryServicesImpl(services, normalizedConnInfo);
+ connectionQueryServices = new ConnectionQueryServicesImpl(services, normalizedConnInfo, info);
}
ConnectionQueryServices prevValue = connectionQueryServicesMap.putIfAbsent(normalizedConnInfo, connectionQueryServices);
if (prevValue != null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cd5826e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
index 661a222..880cc13 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -424,7 +425,10 @@ public class CsvBulkLoadTool extends Configured implements Tool {
job.setMapperClass(CsvToKeyValueMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class);
-
+
+ // initialize credentials to possibily run in a secure env
+ TableMapReduceUtil.initCredentials(job);
+
HTable htable = new HTable(conf, tableName);
// Auto configure partitioner and reducer according to the Main Data table
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cd5826e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
index 4feb7e3..eb701c5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
@@ -23,6 +23,8 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
import javax.annotation.Nullable;
@@ -105,11 +107,19 @@ public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,ImmutableBytes
Configuration conf = context.getConfiguration();
String jdbcUrl = getJdbcUrl(conf);
+ // pass client configuration into driver
+ Properties clientInfos = new Properties();
+ Iterator<Entry<String, String>> iterator = conf.iterator();
+ while(iterator.hasNext()) {
+ Entry<String,String> entry = iterator.next();
+ clientInfos.setProperty(entry.getKey(), entry.getValue());
+ }
+
// This statement also ensures that the driver class is loaded
LOG.info("Connection with driver {} with url {}", PhoenixDriver.class.getName(), jdbcUrl);
try {
- conn = (PhoenixConnection) DriverManager.getConnection(jdbcUrl);
+ conn = (PhoenixConnection) DriverManager.getConnection(jdbcUrl, clientInfos);
} catch (SQLException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cd5826e/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 232a284..977a7e7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -194,17 +194,24 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* cluster.
* @param services base services from where we derive our default configuration
* @param connectionInfo to provide connection information
+ * @param info hbase configuration properties
* @throws SQLException
*/
- public ConnectionQueryServicesImpl(QueryServices services, ConnectionInfo connectionInfo) {
+ public ConnectionQueryServicesImpl(QueryServices services, ConnectionInfo connectionInfo, Properties info) {
super(services);
Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
for (Entry<String,String> entry : services.getProps()) {
config.set(entry.getKey(), entry.getValue());
}
+ if (info != null) {
+ for (Object key : info.keySet()) {
+ config.set((String) key, info.getProperty((String) key));
+ }
+ }
for (Entry<String,String> entry : connectionInfo.asProps()) {
config.set(entry.getKey(), entry.getValue());
}
+
// Without making a copy of the configuration we cons up, we lose some of our properties
// on the server side during testing.
this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);