You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/08/13 02:47:03 UTC
[1/3] git commit: TAJO-947: ColPartitionStoreExec can cause
URISyntaxException due to special characters. (Mai Hai Thanh via hyunsik)
Repository: tajo
Updated Branches:
refs/heads/index_support fa8f037bb -> b7f96fc3c
TAJO-947: ColPartitionStoreExec can cause URISyntaxException due to special characters. (Mai Hai Thanh via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/87e7ba21
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/87e7ba21
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/87e7ba21
Branch: refs/heads/index_support
Commit: 87e7ba21491ac8a5a6a56357d7b4185c94f5dfd6
Parents: ddfc3f3
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 11 14:04:07 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 11 14:04:07 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/util/StringUtils.java | 96 ++++++++++++++++++++
.../HashBasedColPartitionStoreExec.java | 3 +-
.../SortBasedColPartitionStoreExec.java | 7 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 3 +-
.../tajo/engine/query/TestTablePartitions.java | 43 +++++++++
.../TestTablePartitions/lineitemspecial.tbl | 5 +
.../TestTablePartitions/lineitemspecial_ddl.sql | 3 +
.../TestSpecialCharPartitionKeys1.result | 4 +
.../TestSpecialCharPartitionKeys2.result | 3 +
10 files changed, 165 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 182d07d..8788b17 100644
--- a/CHANGES
+++ b/CHANGES
@@ -109,6 +109,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-947: ColPartitionStoreExec can cause URISyntaxException due
+ to special characters. (Mai Hai Thanh via hyunsik)
+
TAJO-999: SequenceFile key class need to be compatible. (jaehwa)
TAJO-994: 'count(distinct x)' function counts first null value. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 41ea153..90391a8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -21,10 +21,12 @@ package org.apache.tajo.util;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.SystemUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.SignalLogger;
import java.util.Arrays;
+import java.util.BitSet;
public class StringUtils {
@@ -180,4 +182,98 @@ public class StringUtils {
public static String unicodeEscapedDelimiter(char c) {
return CharUtils.unicodeEscaped(c);
}
+
+ /**
+ * The following lines of code that deals with escape characters is mostly copied from HIVE's FileUtils.java
+ */
+
+ static BitSet charToEscape = new BitSet(128);
+ static {
+ for (char c = 0; c < ' '; c++) {
+ charToEscape.set(c);
+ }
+
+ /**
+ * ASCII 01-1F are HTTP control characters that need to be escaped.
+ * \u000A and \u000D are \n and \r, respectively.
+ */
+ char[] clist = new char[] {'\u0001', '\u0002', '\u0003', '\u0004',
+ '\u0005', '\u0006', '\u0007', '\u0008', '\u0009', '\n', '\u000B',
+ '\u000C', '\r', '\u000E', '\u000F', '\u0010', '\u0011', '\u0012',
+ '\u0013', '\u0014', '\u0015', '\u0016', '\u0017', '\u0018', '\u0019',
+ '\u001A', '\u001B', '\u001C', '\u001D', '\u001E', '\u001F',
+ '"', '#', '%', '\'', '*', '/', ':', '=', '?', '\\', '\u007F', '{',
+ '[', ']', '^'};
+
+ for (char c : clist) {
+ charToEscape.set(c);
+ }
+
+ if(Shell.WINDOWS){
+ // On windows, following chars need to be escaped as well
+ char [] winClist = {' ', '<','>','|'};
+ for (char c : winClist) {
+ charToEscape.set(c);
+ }
+ }
+ }
+
+ static boolean needsEscaping(char c) {
+ return c >= 0 && c < charToEscape.size() && charToEscape.get(c);
+ }
+
+ public static String escapePathName(String path) {
+ return escapePathName(path, null);
+ }
+
+ /**
+ * Escapes a path name.
+ * @param path The path to escape.
+ * @param defaultPath
+ * The default name for the path, if the given path is empty or null.
+ * @return An escaped path name.
+ */
+ public static String escapePathName(String path, String defaultPath) {
+ if (path == null || path.length() == 0) {
+ if (defaultPath == null) {
+ return "__TAJO_DEFAULT_PARTITION__";
+ } else {
+ return defaultPath;
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < path.length(); i++) {
+ char c = path.charAt(i);
+ if (needsEscaping(c)) {
+ sb.append('%');
+ sb.append(String.format("%1$02X", (int) c));
+ } else {
+ sb.append(c);
+ }
+ }
+ return sb.toString();
+ }
+
+ public static String unescapePathName(String path) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < path.length(); i++) {
+ char c = path.charAt(i);
+ if (c == '%' && i + 2 < path.length()) {
+ int code = -1;
+ try {
+ code = Integer.valueOf(path.substring(i + 1, i + 3), 16);
+ } catch (Exception e) {
+ code = -1;
+ }
+ if (code >= 0) {
+ sb.append((char) code);
+ i += 2;
+ continue;
+ }
+ }
+ sb.append(c);
+ }
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java
index 6cef22e..44d1270 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java
@@ -26,6 +26,7 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.engine.planner.logical.StoreTableNode;
import org.apache.tajo.storage.Appender;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
@@ -79,7 +80,7 @@ public class HashBasedColPartitionStoreExec extends ColPartitionStoreExec {
if(i > 0)
sb.append("/");
sb.append(keyNames[i]).append("=");
- sb.append(datum.asChars());
+ sb.append(StringUtils.escapePathName(datum.asChars()));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java
index d09e296..9ce455f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java
@@ -30,6 +30,7 @@ import org.apache.tajo.engine.planner.logical.StoreTableNode;
import org.apache.tajo.storage.Appender;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
@@ -78,8 +79,8 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec {
if(i > 0) {
sb.append("/");
}
- sb.append(keyNames[i]).append("=");
- sb.append(datum.asChars());
+ sb.append(keyNames[i]).append("=");
+ sb.append(StringUtils.escapePathName(datum.asChars()));
}
return sb.toString();
}
@@ -95,7 +96,7 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec {
appender = getAppender(getSubdirectory(currentKey));
prevKey = new VTuple(currentKey);
} else {
- if (!prevKey.equals(currentKey)) {
+ if (!prevKey.equals(currentKey) && !getSubdirectory(prevKey).equalsIgnoreCase(getSubdirectory(currentKey))) {
appender.close();
StatisticsUtil.aggregateTableStat(aggregated, appender.getStats());
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index f2e47bc..0752e11 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -37,6 +37,7 @@ import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleRange;
import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.util.StringUtils;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@@ -258,7 +259,7 @@ public class TupleUtil {
}
int columnId = partitionColumnSchema.getColumnIdByName(parts[0]);
Column keyColumn = partitionColumnSchema.getColumn(columnId);
- tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), parts[1]));
+ tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), StringUtils.unescapePathName(parts[1])));
}
for (; i < partitionColumnSchema.size(); i++) {
tuple.put(i, NullDatum.get());
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index d9aea53..d80fdb5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -786,4 +786,47 @@ public class TestTablePartitions extends QueryTestCaseBase {
fail("Can't find query from workers" + queryId);
return null;
}
+
+ @Test
+ public final void TestSpecialCharPartitionKeys1() throws Exception {
+ // See - TAJO-947: ColPartitionStoreExec can cause URISyntaxException due to special characters.
+
+ executeDDL("lineitemspecial_ddl.sql", "lineitemspecial.tbl");
+
+ executeString("CREATE TABLE IF NOT EXISTS pTable947 (id int, name text) PARTITION BY COLUMN (type text)")
+ .close();
+ executeString("INSERT OVERWRITE INTO pTable947 SELECT l_orderkey, l_shipinstruct, l_shipmode FROM lineitemspecial")
+ .close();
+ ResultSet res = executeString("select * from pTable947 where type='RA:*?><I/L#%S' or type='AIR'");
+
+ String resStr = resultSetToString(res);
+ String expected =
+ "id,name,type\n" +
+ "-------------------------------\n"
+ + "3,NONE,AIR\n"
+ + "3,TEST SPECIAL CHARS,RA:*?><I/L#%S\n";
+
+ assertEquals(expected, resStr);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void TestSpecialCharPartitionKeys2() throws Exception {
+ // See - TAJO-947: ColPartitionStoreExec can cause URISyntaxException due to special characters.
+
+ executeDDL("lineitemspecial_ddl.sql", "lineitemspecial.tbl");
+
+ executeString("CREATE TABLE IF NOT EXISTS pTable947 (id int, name text) PARTITION BY COLUMN (type text)")
+ .close();
+ executeString("INSERT OVERWRITE INTO pTable947 SELECT l_orderkey, l_shipinstruct, l_shipmode FROM lineitemspecial")
+ .close();
+
+ ResultSet res = executeString("select * from pTable947 where type='RA:*?><I/L#%S'");
+ assertResultSet(res);
+ cleanupQuery(res);
+
+ res = executeString("select * from pTable947 where type='RA:*?><I/L#%S' or type='AIR01'");
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/test/resources/dataset/TestTablePartitions/lineitemspecial.tbl
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestTablePartitions/lineitemspecial.tbl b/tajo-core/src/test/resources/dataset/TestTablePartitions/lineitemspecial.tbl
new file mode 100644
index 0000000..2241213
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestTablePartitions/lineitemspecial.tbl
@@ -0,0 +1,5 @@
+1|DELIVER IN PERSON|TRUCK|
+1|TAKE BACK RETURN|MAIL|
+2|TAKE BACK RETURN|RAIL|
+3|NONE|AIR|
+3|TEST SPECIAL CHARS|RA:*?><I/L#%S|
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql b/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql
new file mode 100644
index 0000000..77e76d5
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql
@@ -0,0 +1,3 @@
+create external table if not exists lineitemspecial (
+ l_orderkey INT4, l_shipinstruct TEXT, l_shipmode TEXT)
+using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path};
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys1.result b/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys1.result
new file mode 100644
index 0000000..98af4ec
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys1.result
@@ -0,0 +1,4 @@
+id,name,type
+-------------------------------
+3,NONE,AIR
+3,TEST SPECIAL CHARS,RA:*?><I/L#%S
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/87e7ba21/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys2.result b/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys2.result
new file mode 100644
index 0000000..d20fff7
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTablePartitions/TestSpecialCharPartitionKeys2.result
@@ -0,0 +1,3 @@
+id,name,type
+-------------------------------
+3,TEST SPECIAL CHARS,RA:*?><I/L#%S
\ No newline at end of file
[2/3] git commit: TAJO-995: HiveMetaStoreClient wrapper should retry
the connection. (jinho)
Posted by ji...@apache.org.
TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
Closes #108
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fbf71668
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fbf71668
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fbf71668
Branch: refs/heads/index_support
Commit: fbf7166897b3b25d8319198216dc4ca529e677ce
Parents: 87e7ba2
Author: jinossy <ji...@gmail.com>
Authored: Mon Aug 11 15:18:29 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Mon Aug 11 15:18:29 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../tajo/catalog/store/HCatalogStore.java | 11 ++++++-----
.../catalog/store/HCatalogStoreClientPool.java | 20 ++++++++++++++++----
.../apache/tajo/catalog/store/HCatalogUtil.java | 6 ++++++
4 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8788b17..839125e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -109,6 +109,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
+
TAJO-947: ColPartitionStoreExec can cause URISyntaxException due
to special characters. (Mai Hai Thanh via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 61bdccd..fb8f4c4 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -48,6 +48,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.thrift.TException;
import java.io.IOException;
import java.util.*;
@@ -77,13 +78,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
@Override
public boolean existTable(final String databaseName, final String tableName) throws CatalogException {
boolean exist = false;
- org.apache.hadoop.hive.ql.metadata.Table table = null;
+ org.apache.hadoop.hive.ql.metadata.Table table;
HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get table
try {
client = clientPool.getClient();
- table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+ table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
if (table != null) {
exist = true;
}
@@ -118,7 +119,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// get hive table schema
try {
client = clientPool.getClient();
- table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+ table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
path = table.getPath();
} catch (NoSuchObjectException nsoe) {
throw new CatalogException("Table not found. - tableName:" + tableName, nsoe);
@@ -291,7 +292,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
client = clientPool.getClient();
return client.getHiveClient().getAllTables(databaseName);
- } catch (MetaException e) {
+ } catch (TException e) {
throw new CatalogException(e);
} finally {
if(client != null) client.release();
@@ -401,7 +402,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
client = clientPool.getClient();
return client.getHiveClient().getAllDatabases();
- } catch (MetaException e) {
+ } catch (TException e) {
throw new CatalogException(e);
} finally {
if (client != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
index 2fff67c..8ccb100 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
@@ -16,7 +16,9 @@ package org.apache.tajo.catalog.store;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.*;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.log4j.Logger;
import java.util.Iterator;
@@ -41,12 +43,22 @@ public class HCatalogStoreClientPool {
* connection pool.
*/
public class HCatalogStoreClient {
- private final HiveMetaStoreClient hiveClient;
+ private final IMetaStoreClient hiveClient;
public AtomicBoolean isInUse = new AtomicBoolean(false);
private HCatalogStoreClient(HiveConf hiveConf) {
try {
- this.hiveClient = new HiveMetaStoreClient(hiveConf);
+ HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+ @Override
+ public HiveMetaHook getHook(Table table) throws MetaException {
+ /* metadata hook implementation, or null if this
+ * storage handler does not need any metadata notifications
+ */
+ return null;
+ }
+ };
+
+ this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
clientPool.add(this);
LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")");
} catch (Exception e) {
@@ -58,7 +70,7 @@ public class HCatalogStoreClientPool {
/**
* Returns the internal HiveMetaStoreClient object.
*/
- public HiveMetaStoreClient getHiveClient() {
+ public IMetaStoreClient getHiveClient() {
return hiveClient;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index 9e60768..54fdb9d 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -20,9 +20,11 @@ package org.apache.tajo.catalog.store;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -30,6 +32,7 @@ import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.thrift.TException;
import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
public class HCatalogUtil {
@@ -137,4 +140,7 @@ public class HCatalogUtil {
}
}
+ public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException {
+ return new Table(client.getTable(dbName, tableName));
+ }
}
[3/3] git commit: Merge branch 'master' of
http://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b7f96fc3
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b7f96fc3
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b7f96fc3
Branch: refs/heads/index_support
Commit: b7f96fc3c11a8f2b465218ffc6e9214ad25b132a
Parents: fa8f037 fbf7166
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Aug 13 09:46:38 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Aug 13 09:46:38 2014 +0900
----------------------------------------------------------------------
CHANGES | 5 +
.../tajo/catalog/store/HCatalogStore.java | 11 ++-
.../catalog/store/HCatalogStoreClientPool.java | 20 +++-
.../apache/tajo/catalog/store/HCatalogUtil.java | 6 ++
.../java/org/apache/tajo/util/StringUtils.java | 96 ++++++++++++++++++++
.../HashBasedColPartitionStoreExec.java | 3 +-
.../SortBasedColPartitionStoreExec.java | 7 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 3 +-
.../tajo/engine/query/TestTablePartitions.java | 43 +++++++++
.../TestTablePartitions/lineitemspecial.tbl | 5 +
.../TestTablePartitions/lineitemspecial_ddl.sql | 3 +
.../TestSpecialCharPartitionKeys1.result | 4 +
.../TestSpecialCharPartitionKeys2.result | 3 +
13 files changed, 195 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7f96fc3/CHANGES
----------------------------------------------------------------------