You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ss...@apache.org on 2016/11/17 20:09:09 UTC
[1/3] phoenix git commit: PHOENIX-3490 PhoenixStorageHandler doesn't
handler hbase configuration properly and don't work in secured env
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 b62ebe0c1 -> 3f1be1a9c
refs/heads/4.x-HBase-1.1 5048af43d -> cd931c0b3
refs/heads/master f1ecd4f7d -> 659671a41
PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work in secured env
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/659671a4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/659671a4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/659671a4
Branch: refs/heads/master
Commit: 659671a411cdbf055ce107d2e83571e66024b4f1
Parents: f1ecd4f
Author: Sergey Soldatov <ss...@apache.org>
Authored: Wed Nov 16 17:07:08 2016 -0800
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Thu Nov 17 12:06:34 2016 -0800
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 55 ++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/659671a4/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index bda2282..a425b7c 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -19,7 +19,10 @@ package org.apache.phoenix.hive;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
@@ -43,6 +47,7 @@ import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -54,6 +59,22 @@ import java.util.Properties;
public class PhoenixStorageHandler extends DefaultStorageHandler implements
HiveStoragePredicateHandler, InputEstimator {
+
+ private Configuration jobConf;
+ private Configuration hbaseConf;
+
+
+ @Override
+ public void setConf(Configuration conf) {
+ jobConf = conf;
+ hbaseConf = HBaseConfiguration.create(conf);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return hbaseConf;
+ }
+
private static final Log LOG = LogFactory.getLog(PhoenixStorageHandler.class);
public PhoenixStorageHandler() {
@@ -67,6 +88,22 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
return new PhoenixMetaHook();
}
+ @Override
+ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+ try {
+ TableMapReduceUtil.addDependencyJars(jobConf);
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+ PhoenixStorageHandler.class);
+ JobConf hbaseJobConf = new JobConf(getConf());
+ org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf);
+ ShimLoader.getHadoopShims().mergeCredentials(jobConf, hbaseJobConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ }
+
@SuppressWarnings("rawtypes")
@Override
public Class<? extends OutputFormat> getOutputFormatClass() {
@@ -167,6 +204,24 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
(PhoenixStorageHandlerConstants.ZOOKEEPER_PORT));
jobProperties.put(HConstants.ZOOKEEPER_ZNODE_PARENT, jobProperties.get
(PhoenixStorageHandlerConstants.ZOOKEEPER_PARENT));
+ addHBaseResources(jobConf, jobProperties);
+ }
+
+ /**
+ * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map
+ * if they are not already present in the jobConf.
+ * @param jobConf Job configuration
+ * @param newJobProperties Map to which new properties should be added
+ */
+ private void addHBaseResources(Configuration jobConf,
+ Map<String, String> newJobProperties) {
+ Configuration conf = new Configuration(false);
+ HBaseConfiguration.addHbaseResources(conf);
+ for (Map.Entry<String, String> entry : conf) {
+ if (jobConf.get(entry.getKey()) == null) {
+ newJobProperties.put(entry.getKey(), entry.getValue());
+ }
+ }
}
@Override
[3/3] phoenix git commit: PHOENIX-3490 PhoenixStorageHandler doesn't
handler hbase configuration properly and don't work in secured env
Posted by ss...@apache.org.
PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work in secured env
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cd931c0b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cd931c0b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cd931c0b
Branch: refs/heads/4.x-HBase-1.1
Commit: cd931c0b304a54019c2c7ea66186005f4f5b5e3e
Parents: 5048af4
Author: Sergey Soldatov <ss...@apache.org>
Authored: Wed Nov 16 17:07:08 2016 -0800
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Thu Nov 17 12:07:38 2016 -0800
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 55 ++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd931c0b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index bda2282..a425b7c 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -19,7 +19,10 @@ package org.apache.phoenix.hive;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
@@ -43,6 +47,7 @@ import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -54,6 +59,22 @@ import java.util.Properties;
public class PhoenixStorageHandler extends DefaultStorageHandler implements
HiveStoragePredicateHandler, InputEstimator {
+
+ private Configuration jobConf;
+ private Configuration hbaseConf;
+
+
+ @Override
+ public void setConf(Configuration conf) {
+ jobConf = conf;
+ hbaseConf = HBaseConfiguration.create(conf);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return hbaseConf;
+ }
+
private static final Log LOG = LogFactory.getLog(PhoenixStorageHandler.class);
public PhoenixStorageHandler() {
@@ -67,6 +88,22 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
return new PhoenixMetaHook();
}
+ @Override
+ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+ try {
+ TableMapReduceUtil.addDependencyJars(jobConf);
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+ PhoenixStorageHandler.class);
+ JobConf hbaseJobConf = new JobConf(getConf());
+ org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf);
+ ShimLoader.getHadoopShims().mergeCredentials(jobConf, hbaseJobConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ }
+
@SuppressWarnings("rawtypes")
@Override
public Class<? extends OutputFormat> getOutputFormatClass() {
@@ -167,6 +204,24 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
(PhoenixStorageHandlerConstants.ZOOKEEPER_PORT));
jobProperties.put(HConstants.ZOOKEEPER_ZNODE_PARENT, jobProperties.get
(PhoenixStorageHandlerConstants.ZOOKEEPER_PARENT));
+ addHBaseResources(jobConf, jobProperties);
+ }
+
+ /**
+ * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map
+ * if they are not already present in the jobConf.
+ * @param jobConf Job configuration
+ * @param newJobProperties Map to which new properties should be added
+ */
+ private void addHBaseResources(Configuration jobConf,
+ Map<String, String> newJobProperties) {
+ Configuration conf = new Configuration(false);
+ HBaseConfiguration.addHbaseResources(conf);
+ for (Map.Entry<String, String> entry : conf) {
+ if (jobConf.get(entry.getKey()) == null) {
+ newJobProperties.put(entry.getKey(), entry.getValue());
+ }
+ }
}
@Override
[2/3] phoenix git commit: PHOENIX-3490 PhoenixStorageHandler doesn't
handler hbase configuration properly and don't work in secured env
Posted by ss...@apache.org.
PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work in secured env
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f1be1a9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f1be1a9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f1be1a9
Branch: refs/heads/4.x-HBase-0.98
Commit: 3f1be1a9cbab3647d3e915d4935855d68180b808
Parents: b62ebe0
Author: Sergey Soldatov <ss...@apache.org>
Authored: Wed Nov 16 17:07:08 2016 -0800
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Thu Nov 17 12:07:02 2016 -0800
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 55 ++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f1be1a9/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index bda2282..a425b7c 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -19,7 +19,10 @@ package org.apache.phoenix.hive;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
@@ -43,6 +47,7 @@ import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -54,6 +59,22 @@ import java.util.Properties;
public class PhoenixStorageHandler extends DefaultStorageHandler implements
HiveStoragePredicateHandler, InputEstimator {
+
+ private Configuration jobConf;
+ private Configuration hbaseConf;
+
+
+ @Override
+ public void setConf(Configuration conf) {
+ jobConf = conf;
+ hbaseConf = HBaseConfiguration.create(conf);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return hbaseConf;
+ }
+
private static final Log LOG = LogFactory.getLog(PhoenixStorageHandler.class);
public PhoenixStorageHandler() {
@@ -67,6 +88,22 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
return new PhoenixMetaHook();
}
+ @Override
+ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+ try {
+ TableMapReduceUtil.addDependencyJars(jobConf);
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+ PhoenixStorageHandler.class);
+ JobConf hbaseJobConf = new JobConf(getConf());
+ org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf);
+ ShimLoader.getHadoopShims().mergeCredentials(jobConf, hbaseJobConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ }
+
@SuppressWarnings("rawtypes")
@Override
public Class<? extends OutputFormat> getOutputFormatClass() {
@@ -167,6 +204,24 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
(PhoenixStorageHandlerConstants.ZOOKEEPER_PORT));
jobProperties.put(HConstants.ZOOKEEPER_ZNODE_PARENT, jobProperties.get
(PhoenixStorageHandlerConstants.ZOOKEEPER_PARENT));
+ addHBaseResources(jobConf, jobProperties);
+ }
+
+ /**
+ * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map
+ * if they are not already present in the jobConf.
+ * @param jobConf Job configuration
+ * @param newJobProperties Map to which new properties should be added
+ */
+ private void addHBaseResources(Configuration jobConf,
+ Map<String, String> newJobProperties) {
+ Configuration conf = new Configuration(false);
+ HBaseConfiguration.addHbaseResources(conf);
+ for (Map.Entry<String, String> entry : conf) {
+ if (jobConf.get(entry.getKey()) == null) {
+ newJobProperties.put(entry.getKey(), entry.getValue());
+ }
+ }
}
@Override