You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2014/05/12 17:33:11 UTC
git commit: TAJO-816: NULL delimiter doesn't apply with
HCatalogStore. (jaehwa)
Repository: tajo
Updated Branches:
refs/heads/master 214301fdd -> 77fdbfb2e
TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/77fdbfb2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/77fdbfb2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/77fdbfb2
Branch: refs/heads/master
Commit: 77fdbfb2e596c7b1c4d54dff9f4161b9b5378889
Parents: 214301f
Author: blrunner <jh...@gruter.com>
Authored: Tue May 13 00:32:59 2014 +0900
Committer: blrunner <jh...@gruter.com>
Committed: Tue May 13 00:32:59 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../tajo/catalog/store/HCatalogStore.java | 20 ++++++++++++--------
.../tajo/catalog/store/TestHCatalogStore.java | 14 ++++++++++----
3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index fcd4dbc..75deb0f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -37,6 +37,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
+
TAJO-823: Missing INET4 handling in DatumFactory.cast(). (jihoon)
TAJO-803: INSERT INTO without FROM throws ClassCastException.
http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 7a694fa..7924af1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -436,7 +436,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// If you want to modify table path, you have to modify on Hive cli.
if (tableDesc.isExternal()) {
table.setTableType(TableType.EXTERNAL_TABLE.name());
- table.getParameters().put("EXTERNAL", "TRUE");
+ table.putToParameters("EXTERNAL", "TRUE");
FileSystem fs = tableDesc.getPath().getFileSystem(conf);
if (fs.isFile(tableDesc.getPath())) {
@@ -479,7 +479,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
- sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+ table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV)) {
@@ -495,14 +495,16 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
- sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+ sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
- sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+ sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
StringEscapeUtils.unescapeJava(fieldDelimiter));
+ table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER);
if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
- sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+ table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
+ table.getParameters().remove(StorageConstants.CSVFILE_NULL);
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
@@ -520,17 +522,19 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
- sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+ sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
- sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+ sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
StringEscapeUtils.unescapeJava(fieldDelimiter));
+ table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
} else {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
- sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+ table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
+ table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index a862baf..729184a 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -174,11 +174,10 @@ public class TestHCatalogStore {
@Test
public void testTableWithNullValue() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
- options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
+ options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+ options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
-
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4);
schema.addColumn("s_name", TajoDataTypes.Type.TEXT);
@@ -191,7 +190,6 @@ public class TestHCatalogStore {
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta,
new Path(warehousePath, new Path(DB_NAME, SUPPLIER)));
-
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, SUPPLIER));
@@ -208,7 +206,15 @@ public class TestHCatalogStore {
assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL),
table1.getMeta().getOption(StorageConstants.CSVFILE_NULL));
+
+ assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER),
+ StringEscapeUtils.escapeJava("\u0002"));
+
+ assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_NULL),
+ StringEscapeUtils.escapeJava("\u0003"));
+
store.dropTable(DB_NAME, SUPPLIER);
+
}
@Test