You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2014/02/12 14:25:38 UTC
git commit: TAJO-592: HCatalogStore should supports RCFile and
default hive field delimiter. (jaehwa)
Updated Branches:
refs/heads/master 9de88cb99 -> 519a16a23
TAJO-592: HCatalogStore should supports RCFile and default hive field delimiter. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/519a16a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/519a16a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/519a16a2
Branch: refs/heads/master
Commit: 519a16a23a4cd7e3eca9959b501297e4be60ca6e
Parents: 9de88cb
Author: blrunner <jh...@gruter.com>
Authored: Wed Feb 12 22:25:29 2014 +0900
Committer: blrunner <jh...@gruter.com>
Committed: Wed Feb 12 22:25:29 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tajo/catalog/store/HCatalogStore.java | 56 +++++++++++++-------
.../apache/tajo/catalog/store/HCatalogUtil.java | 11 ++--
.../tajo/catalog/store/TestHCatalogStore.java | 19 +++++--
4 files changed, 61 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/519a16a2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 65cf31b..e5dde69 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -130,6 +130,8 @@ Release 0.8.0 - unreleased
IMPROVEMENTS
+ TAJO-592: HCatalogStore should supports RCFile and default hive field delimiter. (jaehwa)
+
TAJO-548: Investigate frequent young gc. (Min Zhou via hyunsik)
TAJO-584: Improve distributed merge sort. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/519a16a2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 540ad0f..f796fff 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -43,13 +43,13 @@ import java.util.*;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
public class HCatalogStore extends CatalogConstants implements CatalogStore {
+ public static final String CVSFILE_DELIMITER = "csvfile.delimiter";
+
protected final Log LOG = LogFactory.getLog(getClass());
protected Configuration conf;
private static final int CLIENT_POOL_SIZE = 5;
private final HCatalogStoreClientPool clientPool = new HCatalogStoreClientPool(0);
- private Map<Pair<String, String>, Table> tableMap = new HashMap<Pair<String, String>, Table>();
-
public HCatalogStore(final Configuration conf)
throws InternalException {
this.conf = conf;
@@ -171,17 +171,21 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
if (properties != null) {
// set field delimiter
String fieldDelimiter = "", fileOutputformat = "";
- if (properties.getProperty("field.delim") != null) {
- fieldDelimiter = properties.getProperty("field.delim");
+ if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) {
+ fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM);
+ } else {
+ // if hive table used default row format delimiter, Properties doesn't have it.
+ // So, Tajo must set as follows:
+ fieldDelimiter = "\\001";
}
+
// set file output format
fileOutputformat = properties.getProperty("file.outputformat");
storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat,
fieldDelimiter));
- // TODO: another stored file
- if (storeType.equals(CatalogProtos.StoreType.CSV) && fieldDelimiter != null) {
- options.put("csvfile.delimiter", fieldDelimiter);
+ if (storeType.equals(CatalogProtos.StoreType.CSV) ) {
+ options.put(CVSFILE_DELIMITER, fieldDelimiter);
}
// set data size
@@ -290,8 +294,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
client = clientPool.getClient();
- org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api
- .Table();
+ org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api.Table();
table.setDbName(dbName);
table.setTableName(tableName);
@@ -315,28 +318,41 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
sd.setCols(cols);
- // TODO: compression tyoe
- // n table type
sd.setCompressed(false);
+ if (tableDesc.getMeta().hasParams()) {
+ for (CatalogProtos.KeyValueProto entry: tableDesc.getMeta().getParams().getKeyvalList()) {
+ if (entry.getKey().equals("compression.codec")) {
+ sd.setCompressed(true);
+ } else if (entry.getKey().equals(CVSFILE_DELIMITER)) {
+ sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM, entry.getValue());
+ }
+ }
+ }
sd.setParameters(new HashMap<String, String>());
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(table.getTableName());
- // TODO: another Serialization librarys
- sd.getSerdeInfo().setSerializationLib(
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+ if(tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
+ sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
+ } else {
+ sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+ }
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
// sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
- sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM, "|");
- // TODO: another input format classes
- sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
+ if(tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
+ sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
+ } else {
+ sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
+ }
- // TODO: another output format classes
- sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName
- ());
+ if(tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
+ sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
+ } else {
+ sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
+ }
sd.setSortCols(new ArrayList<Order>());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/519a16a2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index 2c2400a..8310873 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -19,6 +19,8 @@ package org.apache.tajo.catalog.store;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -128,12 +130,13 @@ public class HCatalogUtil {
String inputFormatClass = fileFormatArrary[fileFormatArrary.length-1];
- if(inputFormatClass.equals("HiveIgnoreKeyTextOutputFormat")) {
+ if(inputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) {
return CatalogProtos.StoreType.CSV.name();
+ } else if(inputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) {
+ return CatalogProtos.StoreType.RCFILE.name();
} else {
- //TODO: other file format
- return null;
+ throw new InternalException("Not supported file output format. - file output format:" + fileFormat);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/519a16a2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index 2a6e740..1abf022 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -148,9 +148,16 @@ public class TestHCatalogStore {
sd.getSerdeInfo().setName(tableName);
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
- sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM, "|");
- sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
- sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
+
+ if (tableName.equals(CUSTOMER)) {
+ sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
+ sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
+ sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
+ } else {
+ sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
+ sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
+ }
+
Table table = new Table();
table.setDbName(DB_NAME);
@@ -263,6 +270,8 @@ public class TestHCatalogStore {
assertEquals("c_comment", columns.get(7).getColumnName());
assertEquals(TajoDataTypes.Type.TEXT, columns.get(7).getDataType().getType());
assertNull(table.getPartitionMethod());
+ assertEquals(table.getMeta().getStoreType().name(), CatalogProtos.StoreType.RCFILE.name());
+
table = new TableDesc(store.getTable(DB_NAME + "." + NATION));
columns = table.getSchema().getColumns();
@@ -279,6 +288,10 @@ public class TestHCatalogStore {
assertNotNull(table.getPartitionMethod());
assertEquals("type", table.getPartitionMethod().getExpressionSchema().getColumn(0).getColumnName());
assertEquals(CatalogProtos.PartitionType.COLUMN, table.getPartitionMethod().getPartitionType());
+
+ assertEquals(table.getMeta().getOption(HCatalogStore.CVSFILE_DELIMITER), "\\001");
+ assertEquals(table.getMeta().getStoreType().name(), CatalogProtos.StoreType.CSV.name());
+
}
@Test