You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/13 02:56:27 UTC
[04/11] git commit: DRILL-696: Use standard HBase configuration
name/value in HBaseStoragePluginConfig
DRILL-696: Use standard HBase configuration name/value in HBaseStoragePluginConfig
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6e6c661f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6e6c661f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6e6c661f
Branch: refs/heads/master
Commit: 6e6c661fd8099fb7f385346ba38e2026c84fc67a
Parents: 508cb5d
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Sun May 11 03:19:43 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon May 12 11:46:38 2014 -0700
----------------------------------------------------------------------
.../store/hbase/HBaseStoragePluginConfig.java | 52 +++++++++++---------
.../org/apache/drill/hbase/BaseHBaseTest.java | 2 +-
.../org/apache/drill/hbase/HBaseTestsSuite.java | 13 +++--
.../hbase/hbase_scan_screen_physical.json | 6 ++-
...base_scan_screen_physical_column_select.json | 6 ++-
...base_scan_screen_physical_family_select.json | 6 ++-
.../src/test/resources/storage-plugins.json | 39 ++-------------
distribution/src/resources/storage-plugins.json | 8 +--
8 files changed, 58 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java
index 5a434d6..054d65f 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java
@@ -17,45 +17,43 @@
*/
package org.apache.drill.exec.store.hbase;
+import java.util.Map;
+
import org.apache.drill.common.logical.StoragePluginConfigBase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
@JsonTypeName("hbase")
public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements DrillHBaseConstants {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseStoragePluginConfig.class);
- @JsonProperty
- public String zookeeperQuorum;
-
- @JsonProperty
- public int zookeeperPort;
+ private Map<String, String> config;
+ @JsonIgnore
private Configuration hbaseConf;
- private HConnectionKey hbaseConfKey;
@JsonCreator
- public HBaseStoragePluginConfig(@JsonProperty("zookeeperQuorum") String zookeeperQuorum,
- @JsonProperty("zookeeperPort") int zookeeperPort) {
- this.zookeeperQuorum = zookeeperQuorum;
- this.zookeeperPort = zookeeperPort;
-
- this.hbaseConf = HBaseConfiguration.create();
- logger.debug("Configuring HBase StoragePlugin with zookeeper quorum '{}', port '{}' node '{}'.",
- zookeeperQuorum, zookeeperPort, hbaseConf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
- if (zookeeperQuorum != null && zookeeperQuorum.length() != 0) {
- hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zookeeperQuorum);
- hbaseConf.setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort);
+ public HBaseStoragePluginConfig(@JsonProperty("config") Map<String, String> props) {
+ this.config = props;
+ if (config == null) {
+ config = Maps.newHashMap();
}
- this.hbaseConfKey = new HConnectionKey(hbaseConf);
+ logger.debug("Configuring HBase StoragePlugin with zookeeper quorum '{}', port '{}'.",
+ config.get(HConstants.ZOOKEEPER_QUORUM), config.get(HBASE_ZOOKEEPER_PORT));
+ }
+
+ @JsonProperty
+ public Map<String, String> getConfig() {
+ return ImmutableMap.copyOf(config);
}
@Override
@@ -66,24 +64,32 @@ public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements
return false;
}
HBaseStoragePluginConfig that = (HBaseStoragePluginConfig) o;
- return this.hbaseConfKey.equals(that.hbaseConfKey);
+ return config.equals(that.config);
}
@Override
public int hashCode() {
- return this.hbaseConfKey != null ? this.hbaseConfKey.hashCode() : 0;
+ return this.config != null ? this.config.hashCode() : 0;
}
@JsonIgnore
public Configuration getHBaseConf() {
+ if (hbaseConf == null) {
+ hbaseConf = HBaseConfiguration.create();
+ if (config != null) {
+ for (Map.Entry<String, String> entry : config.entrySet()) {
+ hbaseConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ }
return hbaseConf;
}
@JsonIgnore
@VisibleForTesting
public void setZookeeperPort(int zookeeperPort) {
- this.zookeeperPort = zookeeperPort;
- hbaseConf.setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort);
+ this.config.put(HBASE_ZOOKEEPER_PORT, String.valueOf(zookeeperPort));
+ getHBaseConf().setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index 0753a4d..a68cf70 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -76,7 +76,7 @@ public class BaseHBaseTest extends BaseTestQuery {
protected String getPlanText(String planFile, String tableName) throws IOException {
return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
- .replaceFirst("\"zookeeperPort\".*:.*\\d+", "\"zookeeperPort\" : " + HBaseTestsSuite.getZookeeperPort())
+ .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort())
.replace("[TABLE_NAME]", tableName);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
index 36c31b7..32d7aaa 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
@@ -21,8 +21,6 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -40,7 +38,8 @@ import org.junit.runners.Suite.SuiteClasses;
TestHBaseFilterPushDown.class,
TestHBaseProjectPushDown.class})
public class HBaseTestsSuite {
- private static final Log LOG = LogFactory.getLog(HBaseTestsSuite.class);
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseTestsSuite.class);
+
private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
protected static final String TEST_TABLE_1 = "TestTable1";
@@ -70,11 +69,11 @@ public class HBaseTestsSuite {
}
if (manageHBaseCluster) {
- LOG.info("Starting HBase mini cluster.");
+ logger.info("Starting HBase mini cluster.");
UTIL = new HBaseTestingUtility(conf);
UTIL.startMiniCluster();
hbaseClusterCreated = true;
- LOG.info("HBase mini cluster started.");
+ logger.info("HBase mini cluster started. Zookeeper port: '{}'", getZookeeperPort());
}
admin = new HBaseAdmin(conf);
@@ -104,9 +103,9 @@ public class HBaseTestsSuite {
}
if (hbaseClusterCreated) {
- LOG.info("Shutting down HBase mini cluster.");
+ logger.info("Shutting down HBase mini cluster.");
UTIL.shutdownMiniCluster();
- LOG.info("HBase mini cluster stopped.");
+ logger.info("HBase mini cluster stopped.");
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json
index 17e17e4..7f9015b 100644
--- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json
+++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json
@@ -15,8 +15,10 @@
storage:
{
"type":"hbase",
- "zookeeperQuorum" : "localhost",
- "zookeeperPort" : 2181
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
}
},
{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json
index dc08031..f399f6f 100644
--- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json
+++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json
@@ -15,8 +15,10 @@
storage:
{
"type":"hbase",
- "zookeeperQuorum" : "localhost",
- "zookeeperPort" : 2181
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
},
columns: [
"`f2`.c1", "`f2`.c2", "row_key"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json
index ce027a0..0002164 100644
--- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json
+++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json
@@ -15,8 +15,10 @@
storage:
{
"type":"hbase",
- "zookeeperQuorum" : "localhost",
- "zookeeperPort" : 2181
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
},
columns: [
"f2"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/storage-plugins.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/storage-plugins.json b/contrib/storage-hbase/src/test/resources/storage-plugins.json
index 160583a..0e93f7e 100644
--- a/contrib/storage-hbase/src/test/resources/storage-plugins.json
+++ b/contrib/storage-hbase/src/test/resources/storage-plugins.json
@@ -1,40 +1,11 @@
{
"storage":{
- dfs: {
- type: "file",
- connection: "file:///",
- formats: {
- "psv" : {
- type: "text",
- extensions: [ "tbl" ],
- delimiter: "|"
- },
- "csv" : {
- type: "text",
- extensions: [ "csv" ],
- delimiter: ","
- },
- "tsv" : {
- type: "text",
- extensions: [ "tsv" ],
- delimiter: "\t"
- },
- "parquet" : {
- type: "parquet"
- },
- "json" : {
- type: "json"
- }
- }
- },
- cp: {
- type: "file",
- connection: "classpath:///"
- },
hbase : {
type:"hbase",
- zookeeperQuorum : "localhost",
- zookeeperPort : 2181
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/distribution/src/resources/storage-plugins.json
----------------------------------------------------------------------
diff --git a/distribution/src/resources/storage-plugins.json b/distribution/src/resources/storage-plugins.json
index c3f4a4b..3b1cbd0 100644
--- a/distribution/src/resources/storage-plugins.json
+++ b/distribution/src/resources/storage-plugins.json
@@ -59,9 +59,11 @@
/*,
hbase : {
- type:"hbase",
- zookeeperQuorum : "localhost",
- zookeeperPort : 2181
+ type:"hbase",
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
}
*/
}