You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by fe...@apache.org on 2016/04/01 23:53:15 UTC
incubator-zeppelin git commit: [ZEPPELIN-767] HBase interpreter does
not work with HBase on a remote cluster
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master c79d4379c -> 26a2d641c
[ZEPPELIN-767] HBase interpreter does not work with HBase on a remote cluster
### What is this PR for?
HBase interpreter fails with message "ERROR: KeeperErrorCode = ConnectionLoss for /hbase" when connecting to a remote HBASE (for instance, HBase running in CDH cluster)
Initially it's thought that zkquoram setttings are not getting applied, but deeper investigations reveal that hbase-site.xml cannot be loaded.
HBASE_HOME or HBASE_CONF_DIR is set by `hbase` script when running hbase shell - interpreter will need to at minimum replicate that behavior to add the directory with hbase-site.xml to CLASS_PATH in order to fix this issue.
### What type of PR is it?
Bug Fix
### Todos
* [x] - Bug fix
* [x] - Update documentation
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-767
### How should this be tested?
(tested) Run HBase locally (standalone: https://hbase.apache.org/book.html#quickstart)
(tested) Set HBASE_HOME in env and work with HBASE on a Hadoop cluster
(tested) Set HBASE_CONF_DIR in env and work with HBASE on a Hadoop cluster
### Screenshots (if appropriate)
N/A
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? Added
Author: Felix Cheung <fe...@hotmail.com>
Closes #799 from felixcheung/hbaseconf and squashes the following commits:
ae90626 [Felix Cheung] fix test
a82c2a6 [Felix Cheung] fix bug, add doc, update text
eeb341f [Felix Cheung] set hbase conf dir to classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/26a2d641
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/26a2d641
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/26a2d641
Branch: refs/heads/master
Commit: 26a2d641c70b1a96b9276ea5e48504e0911f62cf
Parents: c79d437
Author: Felix Cheung <fe...@hotmail.com>
Authored: Sun Mar 27 22:04:23 2016 -0700
Committer: Felix Cheung <fe...@apache.org>
Committed: Fri Apr 1 14:53:04 2016 -0700
----------------------------------------------------------------------
bin/interpreter.sh | 8 ++++
conf/zeppelin-env.sh.template | 6 +++
docs/interpreter/hbase.md | 23 ++++++++++-
.../apache/zeppelin/hbase/HbaseInterpreter.java | 43 +++++++++++++++-----
.../zeppelin/hbase/HbaseInterpreterTest.java | 4 +-
5 files changed, 69 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 18e1540..69c94f6 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -132,6 +132,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
fi
+elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
+ if [[ -n "${HBASE_CONF_DIR}" ]]; then
+ ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}"
+ elif [[ -n "${HBASE_HOME}" ]]; then
+ ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf"
+ else
+ echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded"
+ fi
fi
addJarInDir "${LOCAL_INTERPRETER_REPO}"
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index 638162c..6279de7 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -64,3 +64,9 @@
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
+#### HBase interpreter configuration ####
+
+## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
+
+# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be
+# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/docs/interpreter/hbase.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md
index 7766b02..2eaa915 100644
--- a/docs/interpreter/hbase.md
+++ b/docs/interpreter/hbase.md
@@ -34,7 +34,7 @@ mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr
<tr>
<td>hbase.home</td>
<td>/usr/lib/hbase</td>
- <td>Installation directory of Hbase</td>
+ <td>Installation directory of HBase, defaults to HBASE_HOME in environment</td>
</tr>
<tr>
<td>hbase.ruby.sources</td>
@@ -42,12 +42,31 @@ mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr
<td>Path to Ruby scripts relative to 'hbase.home'</td>
</tr>
<tr>
- <td>hbase.test.mode</td>
+ <td>zeppelin.hbase.test.mode</td>
<td>false</td>
<td>Disable checks for unit and manual tests</td>
</tr>
</table>
+If you want to connect to HBase running on a cluster, you'll need to follow the next step.
+
+### Export HBASE_HOME
+In **conf/zeppelin-env.sh**, export `HBASE_HOME` environment variable with your HBase installation path. This ensures `hbase-site.xml` can be loaded.
+
+for example
+
+```bash
+export HBASE_HOME=/usr/lib/hbase
+```
+
+or, when running with CDH
+
+```bash
+export HBASE_HOME="/opt/cloudera/parcels/CDH/lib/hbase"
+```
+
+You can optionally export `HBASE_CONF_DIR` instead of `HBASE_HOME` should you have custom HBase configurations.
+
## Enabling the HBase Shell Interpreter
In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**.
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
index dbcb33d..84e4105 100644
--- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
+++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
@@ -37,21 +37,20 @@ import java.util.List;
import java.util.Properties;
/**
- * Support for Hbase Shell. All the commands documented here
+ * Support for HBase Shell. All the commands documented here
* http://hbase.apache.org/book.html#shell is supported.
*
* Requirements:
- * Hbase Shell should be installed on the same machine. To be more specific, the following dir.
+ * HBase Shell should be installed on the same machine. To be more specific, the following dir.
* should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
- * Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure
+ * HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure
* that the client is configured properly.
*
* The interpreter takes 3 config parameters:
- * hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/
+ * hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/
* hbase.ruby.sources: Dir where shell ruby code is installed.
* Path is relative to hbase.home. Default: lib/ruby
- * hbase.irb.load: (Testing only) Default is true.
- * Whether to load irb in the interpreter.
+ * zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
*/
public class HbaseInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
@@ -62,11 +61,13 @@ public class HbaseInterpreter extends Interpreter {
static {
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
new InterpreterPropertyBuilder()
- .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
+ .add("hbase.home",
+ getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
+ "Installation directory of HBase")
.add("hbase.ruby.sources", "lib/ruby",
"Path to Ruby scripts relative to 'hbase.home'")
- .add("hbase.test.mode", "false", "Disable checks for unit and manual tests")
- .build());
+ .add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
+ .build());
}
public HbaseInterpreter(Properties property) {
@@ -79,7 +80,7 @@ public class HbaseInterpreter extends Interpreter {
this.writer = new StringWriter();
scriptingContainer.setOutput(this.writer);
- if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) {
+ if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
String hbase_home = getProperty("hbase.home");
String ruby_src = getProperty("hbase.ruby.sources");
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
@@ -89,7 +90,7 @@ public class HbaseInterpreter extends Interpreter {
File f = abs_ruby_src.toFile();
if (!f.exists() || !f.isDirectory()) {
- throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src
+ throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src
+ "'");
}
@@ -155,4 +156,24 @@ public class HbaseInterpreter extends Interpreter {
return null;
}
+ private static String getSystemDefault(
+ String envName,
+ String propertyName,
+ String defaultValue) {
+
+ if (envName != null && !envName.isEmpty()) {
+ String envValue = System.getenv().get(envName);
+ if (envValue != null) {
+ return envValue;
+ }
+ }
+
+ if (propertyName != null && !propertyName.isEmpty()) {
+ String propValue = System.getProperty(propertyName);
+ if (propValue != null) {
+ return propValue;
+ }
+ }
+ return defaultValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
index e218070..8c5e29c 100644
--- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
+++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
@@ -40,7 +40,7 @@ public class HbaseInterpreterTest {
Properties properties = new Properties();
properties.put("hbase.home", "");
properties.put("hbase.ruby.sources", "");
- properties.put("hbase.test.mode", "true");
+ properties.put("zeppelin.hbase.test.mode", "true");
hbaseInterpreter = new HbaseInterpreter(properties);
hbaseInterpreter.open();
@@ -72,4 +72,4 @@ public class HbaseInterpreterTest {
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
}
-}
\ No newline at end of file
+}