You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/01/20 14:04:19 UTC
[1/5] tajo git commit: TAJO-1225: Fix wrong schema name in JDBC
driver. (jaehwa)
Repository: tajo
Updated Branches:
refs/heads/index_support 5866c36de -> a72ef4a57
TAJO-1225: Fix wrong schema name in JDBC driver. (jaehwa)
Closes #330
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/76ece8b8
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/76ece8b8
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/76ece8b8
Branch: refs/heads/index_support
Commit: 76ece8b874f4a0c5554e784d7bfcc1474b217c47
Parents: 1bf71f6
Author: JaeHwa Jung <bl...@apache.org>
Authored: Mon Jan 19 10:44:03 2015 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Mon Jan 19 10:44:03 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
tajo-common/src/main/java/org/apache/tajo/TajoConstants.java | 2 +-
.../results/TestTajoDatabaseMetaData/getSchemas1.result | 2 +-
.../results/TestTajoDatabaseMetaData/getSchemas2.result | 2 +-
.../resources/results/TestTajoDatabaseMetaData/getTables1.result | 4 ++--
.../resources/results/TestTajoDatabaseMetaData/getTables2.result | 4 ++--
6 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index e33ca6f..212131d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -164,6 +164,8 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1225: Fix wrong schema name in JDBC driver. (jaehwa)
+
TAJO-1304: Can not found TextFile in catalog. (jinho)
TAJO-1299: TB and PB representations in StorageUnit are overflow.
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
index 15336bb..e6fc89b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
@@ -29,7 +29,7 @@ public class TajoConstants {
public static final String DEFAULT_TABLESPACE_NAME = "default";
public static final String DEFAULT_DATABASE_NAME = "default";
- public static final String DEFAULT_SCHEMA_NAME = "public";
+ public static final String DEFAULT_SCHEMA_NAME = "";
/** Java 6 only recognize GMT instead of UTC. So, we should keep using GMT. */
public static final String DEFAULT_SYSTEM_TIMEZONE = "GMT";
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas1.result b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas1.result
index f922ad7..0a013d9 100644
--- a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas1.result
+++ b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas1.result
@@ -1,3 +1,3 @@
TABLE_SCHEM,TABLE_CATALOG
-------------------------------
-public,jdbc_test1
\ No newline at end of file
+,jdbc_test1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas2.result b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas2.result
index f922ad7..0a013d9 100644
--- a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas2.result
+++ b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getSchemas2.result
@@ -1,3 +1,3 @@
TABLE_SCHEM,TABLE_CATALOG
-------------------------------
-public,jdbc_test1
\ No newline at end of file
+,jdbc_test1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables1.result b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables1.result
index 2ed02c8..201d325 100644
--- a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables1.result
+++ b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables1.result
@@ -1,4 +1,4 @@
TABLE_CAT,TABLE_SCHEM,TABLE_NAME,TABLE_TYPE,REMARKS
-------------------------------
-jdbc_test3,public,table1,TABLE,
-jdbc_test3,public,table2,TABLE,
\ No newline at end of file
+jdbc_test3,,table1,TABLE,
+jdbc_test3,,table2,TABLE,
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/76ece8b8/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables2.result b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables2.result
index 4c8d39f..65cde70 100644
--- a/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables2.result
+++ b/tajo-core/src/test/resources/results/TestTajoDatabaseMetaData/getTables2.result
@@ -1,4 +1,4 @@
TABLE_CAT,TABLE_SCHEM,TABLE_NAME,TABLE_TYPE,REMARKS
-------------------------------
-Jdbc_Test4,public,table3,TABLE,
-Jdbc_Test4,public,table4,TABLE,
\ No newline at end of file
+Jdbc_Test4,,table3,TABLE,
+Jdbc_Test4,,table4,TABLE,
\ No newline at end of file
[5/5] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a72ef4a5
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a72ef4a5
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a72ef4a5
Branch: refs/heads/index_support
Commit: a72ef4a575c45b7cc344e3b1a2f193d7d92ce4e4
Parents: 5866c36 4d1006d
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Jan 20 22:04:08 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Jan 20 22:04:08 2015 +0900
----------------------------------------------------------------------
CHANGES | 11 +-
.../java/org/apache/tajo/TajoConstants.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 18 +-
.../tajo/engine/query/TestHBaseTable.java | 20 ++
.../apache/tajo/engine/query/TestSortQuery.java | 28 +++
.../TestTajoDatabaseMetaData/getSchemas1.result | 2 +-
.../TestTajoDatabaseMetaData/getSchemas2.result | 2 +-
.../TestTajoDatabaseMetaData/getTables1.result | 4 +-
.../TestTajoDatabaseMetaData/getTables2.result | 4 +-
tajo-dist/src/main/bin/tajo | 26 +++
tajo-dist/src/main/conf/tajo-env.sh | 5 +-
tajo-docs/src/main/sphinx/hbase_integration.rst | 181 +++++++++++++++++++
tajo-docs/src/main/sphinx/index.rst | 3 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 9 +-
14 files changed, 298 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[2/5] tajo git commit: TAJO-1290: Add HBase Storage Integration
Documentation. (jaehwa)
Posted by ji...@apache.org.
TAJO-1290: Add HBase Storage Integration Documentation. (jaehwa)
Closes #352
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9d749d6b
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9d749d6b
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9d749d6b
Branch: refs/heads/index_support
Commit: 9d749d6b798d8a3cfb5c11f0b6ad9e8f98c52fd4
Parents: 76ece8b
Author: JaeHwa Jung <bl...@apache.org>
Authored: Mon Jan 19 14:20:25 2015 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Mon Jan 19 14:20:25 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
tajo-docs/src/main/sphinx/hbase_integration.rst | 181 +++++++++++++++++++
tajo-docs/src/main/sphinx/index.rst | 3 +-
3 files changed, 185 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/9d749d6b/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 212131d..1be019a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,6 +27,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1290: Add HBase Storage Integration Documentation. (jaehwa)
+
TAJO-1293: Tajo have to accept hostname beginning with digits.
(Jinhang Choi via jihun)
http://git-wip-us.apache.org/repos/asf/tajo/blob/9d749d6b/tajo-docs/src/main/sphinx/hbase_integration.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/hbase_integration.rst b/tajo-docs/src/main/sphinx/hbase_integration.rst
new file mode 100644
index 0000000..73ef6d1
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/hbase_integration.rst
@@ -0,0 +1,181 @@
+*************************************
+HBase Integration
+*************************************
+
+Apache Tajo™ storage supports integration with Apache HBase™.
+This integration allows Tajo to access all tables used in Apache HBase.
+
+In order to use this feature, you need to build add some configs into ``conf/tajo-env.sh`` and then add some properties into a table create statement.
+
+This section describes how to setup HBase integration.
+
+First, you need to set your HBase home directory to the environment variable ``HBASE_HOME`` in conf/tajo-env.sh as follows: ::
+
+ export HBASE_HOME=/path/to/your/hbase/directory
+
+If you set the directory, Tajo will add HBase library file to classpath.
+
+
+
+========================
+CREATE TABLE
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ CREATE [EXTERNAL] TABLE [IF NOT EXISTS] <table_name> [(<column_name> <data_type>, ... )]
+ USING hbase
+ WITH ('table'='<hbase_table_name>'
+ , 'columns'=':key,<column_family_name>:<qualifier_name>, ...'
+ , 'hbase.zookeeper.quorum'='<zookeeper_address>')
+
+Options
+
+* ``table`` : Set hbase origin table name. If you want to create an external table, the table must exists on HBase. The other way, if you want to create a managed table, the table must doesn't exist on HBase.
+* ``columns`` : :key means HBase row key. The number of columns entry need to equals to the number of Tajo table column
+* ``hbase.zookeeper.quorum`` : Set zookeeper quorum address. You can use different zookeeper cluster on the same Tajo database. If you don't set the zookeeper address, Tajo will refer the property of hbase-site.xml file.
+
+
+``IF NOT EXISTS`` allows ``CREATE [EXTERNAL] TABLE`` statement to avoid an error which occurs when the table does not exist.
+
+
+
+========================
+ DROP TABLE
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ DROP TABLE [IF EXISTS] <table_name> [PURGE]
+
+``IF EXISTS`` allows ``DROP TABLE`` statement to avoid an error which occurs when the table does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents on HBase cluster. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents on HBase cluster.
+
+
+========================
+INSERT (OVERWRITE) INTO
+========================
+
+INSERT OVERWRITE statement overwrites a table data of an existing table. Tajo's INSERT OVERWRITE statement follows ``INSERT INTO SELECT`` statement of SQL. The examples are as follows:
+
+.. code-block:: sql
+
+ -- when a target table schema and output schema are equivalent to each other
+ INSERT OVERWRITE INTO t1 SELECT l_orderkey, l_partkey, l_quantity FROM lineitem;
+ -- or
+ INSERT OVERWRITE INTO t1 SELECT * FROM lineitem;
+
+ -- when the output schema are smaller than the target table schema
+ INSERT OVERWRITE INTO t1 SELECT l_orderkey FROM lineitem;
+
+ -- when you want to specify certain target columns
+ INSERT OVERWRITE INTO t1 (col1, col3) SELECT l_orderkey, l_quantity FROM lineitem;
+
+
+.. note::
+
+ If you don't set row key option, You are never able to use your table data. Because Tajo need to have some key columns for sorting before creating result data.
+
+
+
+========================
+Usage
+========================
+
+In order to create a new HBase table which is to be managed by Tajo, use the USING clause on CREATE TABLE:
+
+.. code-block:: sql
+
+ CREATE EXTERNAL TABLE blog (rowkey text, author text, register_date text, title text)
+ USING hbase WITH (
+ 'table'='blog'
+ , 'columns'=':key,info:author,info:date,content:title');
+
+After executing the command above, you should be able to see the new table in the HBase shell:
+
+.. code-block:: sql
+
+ $ hbase shell
+ create 'blog', {NAME=>'info'}, {NAME=>'content'}
+ put 'blog', 'hyunsik-02', 'content:title', 'Getting started with Tajo on your desktop'
+ put 'blog', 'hyunsik-02', 'info:author', 'Hyunsik Choi'
+ put 'blog', 'hyunsik-02', 'info:date', '2014-12-03'
+ put 'blog', 'blrunner-01', 'content:title', 'Apache Tajo: A Big Data Warehouse System on Hadoop'
+ put 'blog', 'blrunner-01', 'info:author', 'Jaehwa Jung'
+ put 'blog', 'blrunner-01', 'info:date', '2014-10-31'
+ put 'blog', 'jhkim-01', 'content:title', 'APACHE TAJO™ v0.9 HAS ARRIVED!'
+ put 'blog', 'jhkim-01', 'info:author', 'Jinho Kim'
+ put 'blog', 'jhkim-01', 'info:date', '2014-10-22'
+
+And then create the table and query the table meta data with ``\d`` option:
+
+.. code-block:: sql
+
+ default> \d blog;
+
+ table name: default.blog
+ table path:
+ store type: HBASE
+ number of rows: unknown
+ volume: 0 B
+ Options:
+ 'columns'=':key,info:author,info:date,content:title'
+ 'table'='blog'
+
+ schema:
+ rowkey TEXT
+ author TEXT
+ register_date TEXT
+ title TEXT
+
+
+And then query the table as follows:
+
+.. code-block:: sql
+
+ default> SELECT * FROM blog;
+ rowkey, author, register_date, title
+ -------------------------------
+ blrunner-01, Jaehwa Jung, 2014-10-31, Apache Tajo: A Big Data Warehouse System on Hadoop
+ hyunsik-02, Hyunsik Choi, 2014-12-03, Getting started with Tajo on your desktop
+ jhkim-01, Jinho Kim, 2014-10-22, APACHE TAJO™ v0.9 HAS ARRIVED!
+
+ default> SELECT * FROM blog WHERE rowkey = 'blrunner-01';
+ Progress: 100%, response time: 2.043 sec
+ rowkey, author, register_date, title
+ -------------------------------
+ blrunner-01, Jaehwa Jung, 2014-10-31, Apache Tajo: A Big Data Warehouse System on Hadoop
+
+
+Here's how to insert data the HBase table:
+
+.. code-block:: sql
+
+ CREATE TABLE blog_backup(rowkey text, author text, register_date text, title text)
+ USING hbase WITH (
+ 'table'='blog_backup'
+ , 'columns'=':key,info:author,info:date,content:title');
+ INSERT OVERWRITE INTO blog_backup SELECT * FROM blog;
+
+
+Use HBase shell to verify that the data actually got loaded:
+
+.. code-block:: sql
+
+ hbase(main):004:0> scan 'blog_backup'
+ ROW COLUMN+CELL
+ blrunner-01 column=content:title, timestamp=1421227531054, value=Apache Tajo: A Big Data Warehouse System on Hadoop
+ blrunner-01 column=info:author, timestamp=1421227531054, value=Jaehwa Jung
+ blrunner-01 column=info:date, timestamp=1421227531054, value=2014-10-31
+ hyunsik-02 column=content:title, timestamp=1421227531054, value=Getting started with Tajo on your desktop
+ hyunsik-02 column=info:author, timestamp=1421227531054, value=Hyunsik Choi
+ hyunsik-02 column=info:date, timestamp=1421227531054, value=2014-12-03
+ jhkim-01 column=content:title, timestamp=1421227531054, value=APACHE TAJO\xE2\x84\xA2 v0.9 HAS ARRIVED!
+ jhkim-01 column=info:author, timestamp=1421227531054, value=Jinho Kim
+ jhkim-01 column=info:date, timestamp=1421227531054, value=2014-10-22
+ 3 row(s) in 0.0470 seconds
+
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/9d749d6b/tajo-docs/src/main/sphinx/index.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst
index 667f270..1222f54 100644
--- a/tajo-docs/src/main/sphinx/index.rst
+++ b/tajo-docs/src/main/sphinx/index.rst
@@ -40,7 +40,8 @@ Table of Contents:
index_overview
backup_and_restore
hcatalog_integration
- jdbc_driver
+ hbase_integration
+ jdbc_driver
tajo_client_api
faq
[3/5] tajo git commit: TAJO-1307: HBaseStorageManager need to support
for users to use hbase-site.xml file. (jaehwa)
Posted by ji...@apache.org.
TAJO-1307: HBaseStorageManager need to support for users to use hbase-site.xml file. (jaehwa)
Closes #350
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6f07cefd
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6f07cefd
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6f07cefd
Branch: refs/heads/index_support
Commit: 6f07cefd224c8da420ec7ec37974917993a05fc5
Parents: 9d749d6
Author: JaeHwa Jung <bl...@apache.org>
Authored: Mon Jan 19 17:50:26 2015 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Mon Jan 19 17:50:26 2015 +0900
----------------------------------------------------------------------
CHANGES | 5 +++-
.../tajo/engine/query/TestHBaseTable.java | 20 +++++++++++++++
tajo-dist/src/main/bin/tajo | 26 ++++++++++++++++++++
tajo-dist/src/main/conf/tajo-env.sh | 5 +++-
.../tajo/storage/hbase/HBaseStorageManager.java | 9 ++++++-
5 files changed, 62 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f07cefd/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 1be019a..8acd4e5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,7 +1,7 @@
Tajo Change Log
-Release 0.9.1 - unreleased
+Release 0.10.0 - unreleased
NEW FEATURES
@@ -27,6 +27,9 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1307: HBaseStorageManager need to support for users to use
+ hbase-site.xml file. (jaehwa)
+
TAJO-1290: Add HBase Storage Integration Documentation. (jaehwa)
TAJO-1293: Tajo have to accept hostname beginning with digits.
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f07cefd/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
index 520c9f6..741a807 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
@@ -111,6 +111,26 @@ public class TestHBaseTable extends QueryTestCaseBase {
}
}
+
+ @Test
+ public void testVerifyCreateHBaseTableWithHBaseConfiguration() throws Exception {
+ String hostName = InetAddress.getLocalHost().getHostName();
+ String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ assertNotNull(zkPort);
+
+ try {
+ testingCluster.getHBaseUtil().getConf().set(HConstants.ZOOKEEPER_QUORUM, hostName);
+
+ executeString("CREATE TABLE hbase_mapped_table3 (col1 text, col2 text) " +
+ "USING hbase " +
+ "WITH ('table'='hbase_mapped_table3', 'columns'='col1:,col2:')").close();
+
+ testingCluster.getHBaseUtil().getConf().set(HConstants.ZOOKEEPER_QUORUM, null);
+ } catch (Exception e) {
+ assertFalse(e.getMessage().indexOf("HBase mapped table") >= 0);
+ }
+ }
+
@Test
public void testCreateHBaseTable() throws Exception {
String hostName = InetAddress.getLocalHost().getHostName();
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f07cefd/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index e73c6b2..f10f393 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -287,6 +287,32 @@ fi
# Hive Home Configuration End
##############################################################################
+
+##############################################################################
+# Find and Set HBase CLASSPATH
+##############################################################################
+
+HBASE_CONF=$HBASE_HOME/conf/hbase-site.xml
+
+if [ -d ${HBASE_CONF} ]; then
+ CLASSPATH=${HBASE_CONF}:${CLASSPATH}
+fi
+
+
+HBASE_LIB=$HBASE_HOME/lib
+
+if [ -d ${HBASE_LIB} ]; then
+
+ for f in ${HBASE_LIB}/hbase-common-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+fi
+
+##############################################################################
+# Find and Set HBase CLASSPATH
+##############################################################################
+
+
##############################################################################
# Find and Set Tajo CLASSPATH
##############################################################################
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f07cefd/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index 064a82f..4a8e2b2 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -80,4 +80,7 @@ export TAJO_WORKER_STANDBY_MODE=true
# export HIVE_JDBC_DRIVER_DIR=
# Tajo PullServer mode. the default mode is standalone mode
-# export TAJO_PULLSERVER_STANDALONE=false
\ No newline at end of file
+# export TAJO_PULLSERVER_STANDALONE=false
+
+# It must be required to use HBase
+# export HBASE_HOME=
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f07cefd/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index c606e88..c51e31c 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -299,7 +299,14 @@ public class HBaseStorageManager extends StorageManager {
* @throws java.io.IOException
*/
public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException {
- String zkQuorum = tableMeta.getOption(HBaseStorageConstants.META_ZK_QUORUM_KEY, "");
+ boolean hasZkQuorum = tableMeta.containsOption(HBaseStorageConstants.META_ZK_QUORUM_KEY);
+ String zkQuorum = null;
+ if (hasZkQuorum) {
+ zkQuorum = tableMeta.getOption(HBaseStorageConstants.META_ZK_QUORUM_KEY, "");
+ } else {
+ zkQuorum = conf.get(HBaseStorageConstants.META_ZK_QUORUM_KEY, "");
+ }
+
if (zkQuorum == null || zkQuorum.trim().isEmpty()) {
throw new IOException("HBase mapped table is required a '" +
HBaseStorageConstants.META_ZK_QUORUM_KEY + "' attribute.");
[4/5] tajo git commit: TAJO-1257: ORDER BY with NULL FIRST misses
some data. (Keuntae Park)
Posted by ji...@apache.org.
TAJO-1257: ORDER BY with NULL FIRST misses some data. (Keuntae Park)
Closes #307
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/4d1006d5
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/4d1006d5
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/4d1006d5
Branch: refs/heads/index_support
Commit: 4d1006d56b7761500e16afe7be7826422850555a
Parents: 6f07cef
Author: Keuntae Park <si...@apache.org>
Authored: Tue Jan 20 15:47:53 2015 +0900
Committer: Keuntae Park <si...@apache.org>
Committed: Tue Jan 20 15:47:53 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../org/apache/tajo/engine/utils/TupleUtil.java | 18 ++++++++-----
.../apache/tajo/engine/query/TestSortQuery.java | 28 ++++++++++++++++++++
3 files changed, 42 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/4d1006d5/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8acd4e5..eec490b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -169,6 +169,8 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1257: ORDER BY with NULL FIRST misses some data. (Keuntae Park)
+
TAJO-1225: Fix wrong schema name in JDBC driver. (jaehwa)
TAJO-1304: Can not found TextFile in catalog. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/4d1006d5/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 3bb1b5b..027da50 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -97,13 +97,19 @@ public class TupleUtil {
continue;
}
if (columnStat.hasNullValue()) {
- int rangeIndex = sortSpecs[i].isAscending() ? ranges.length - 1 : 0;
- VTuple rangeTuple = sortSpecs[i].isAscending() ? (VTuple) ranges[rangeIndex].getEnd() : (VTuple) ranges[rangeIndex].getStart();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Set null into range: " + col.getQualifiedName() + ", previous tuple is " + rangeTuple);
+ if (sortSpecs[i].isNullFirst()) {
+ int rangeIndex = 0;
+ VTuple rangeTuple = (VTuple) ranges[rangeIndex].getStart();
+ rangeTuple.put(i, NullDatum.get());
+ } else {
+ int rangeIndex = sortSpecs[i].isAscending() ? ranges.length - 1 : 0;
+ VTuple rangeTuple = sortSpecs[i].isAscending() ? (VTuple) ranges[rangeIndex].getEnd() : (VTuple) ranges[rangeIndex].getStart();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Set null into range: " + col.getQualifiedName() + ", previous tuple is " + rangeTuple);
+ }
+ rangeTuple.put(i, NullDatum.get());
+ LOG.info("Set null into range: " + col.getQualifiedName() + ", current tuple is " + rangeTuple);
}
- rangeTuple.put(i, NullDatum.get());
- LOG.info("Set null into range: " + col.getQualifiedName() + ", current tuple is " + rangeTuple);
}
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4d1006d5/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 21a2cc0..9c911f9 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -237,6 +237,34 @@ public class TestSortQuery extends QueryTestCaseBase {
}
@Test
+ public final void testSortOnNullColumn3() throws Exception {
+ KeyValueSet tableOptions = new KeyValueSet();
+ tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N");
+
+ Schema schema = new Schema();
+ schema.addColumn("id", Type.INT4);
+ schema.addColumn("name", Type.TEXT);
+ String[] data = new String[]{ "1|111", "2|\\N", "3|333" };
+ TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1);
+
+ try {
+ ResultSet res = executeString("select * from table11 order by name null first");
+ String ascExpected = "id,name\n" +
+ "-------------------------------\n" +
+ "2,null\n" +
+ "1,111\n" +
+ "3,333\n";
+
+ assertEquals(ascExpected, resultSetToString(res));
+ res.close();
+
+ } finally {
+ executeString("DROP TABLE table11 PURGE");
+ }
+ }
+
+ @Test
public final void testSortOnUnicodeTextAsc() throws Exception {
try {
testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");