You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by kh...@apache.org on 2012/06/10 07:09:24 UTC
svn commit: r1348538 - in /incubator/hcatalog/trunk: CHANGES.txt
src/java/org/apache/hcatalog/common/HCatUtil.java
src/java/org/apache/hcatalog/data/schema/HCatSchema.java
src/test/org/apache/hcatalog/common/TestHCatUtil.java
Author: khorgath
Date: Sun Jun 10 07:09:24 2012
New Revision: 1348538
URL: http://svn.apache.org/viewvc?rev=1348538&view=rev
Log:
HCATALOG-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford via khorgath)
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sun Jun 10 07:09:24 2012
@@ -45,6 +45,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-423 HCatalog should include SerDe-reported fields in the table schema (traviscrawford via khorgath)
+
HCAT-362 add --local-infile to the mysql command (arpitgupta via daijy)
HCAT-417 List Ant 1.8 in install pre-reqs (jghoman via daijy)
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Sun Jun 10 07:09:24 2012
@@ -184,10 +184,9 @@ public class HCatUtil {
return client.getTable(dbName, tableName);
}
- public static HCatSchema getTableSchemaWithPtnCols(Table table)
- throws IOException {
- HCatSchema tableSchema = extractSchemaFromStorageDescriptor(table
- .getSd());
+ public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
+ HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(
+ new org.apache.hadoop.hive.ql.metadata.Table(table).getCols()));
if (table.getPartitionKeys().size() != 0) {
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java Sun Jun 10 07:09:24 2012
@@ -157,4 +157,15 @@ public class HCatSchema implements Seria
}
return sb.toString();
}
+
+ @Override
+ public boolean equals(Object other) {
+ return !((other == null) || !(other instanceof HCatSchema))
+ && this.hashCode() == other.hashCode();
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
}
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java?rev=1348538&r1=1348537&r2=1348538&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/common/TestHCatUtil.java Sun Jun 10 07:09:24 2012
@@ -17,17 +17,31 @@
*/
package org.apache.hcatalog.common;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import junit.framework.TestCase;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hcatalog.common.HCatUtil;
-
-public class TestHCatUtil extends TestCase{
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.junit.Assert;
+import org.junit.Test;
+public class TestHCatUtil {
+ @Test
public void testFsPermissionOperation(){
HashMap<String,Integer> permsCode = new HashMap<String,Integer>();
@@ -43,7 +57,7 @@ public class TestHCatUtil extends TestCa
Integer code = (((i*8)+j)*8)+k;
String perms = (new FsPermission(Short.decode(sb.toString()))).toString();
if (permsCode.containsKey(perms)){
- assertEquals("permissions(" + perms + ") mapped to multiple codes",code,permsCode.get(perms));
+ Assert.assertEquals("permissions(" + perms + ") mapped to multiple codes", code, permsCode.get(perms));
}
permsCode.put(perms, code);
assertFsPermissionTransformationIsGood(perms);
@@ -53,9 +67,10 @@ public class TestHCatUtil extends TestCa
}
private void assertFsPermissionTransformationIsGood(String perms) {
- assertEquals(perms,FsPermission.valueOf("-"+perms).toString());
+ Assert.assertEquals(perms, FsPermission.valueOf("-" + perms).toString());
}
+ @Test
public void testValidateMorePermissive(){
assertConsistentFsPermissionBehaviour(FsAction.ALL,true,true,true,true,true,true,true,true);
assertConsistentFsPermissionBehaviour(FsAction.READ,false,true,false,true,false,false,false,false);
@@ -73,27 +88,91 @@ public class TestHCatUtil extends TestCa
boolean versusX, boolean versusR, boolean versusW,
boolean versusRX, boolean versusRW, boolean versusWX){
- assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL));
- assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE));
- assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE));
- assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ));
- assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE));
- assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE));
- assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE));
- assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE));
+ Assert.assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL));
+ Assert.assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE));
+ Assert.assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE));
+ Assert.assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ));
+ Assert.assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE));
+ Assert.assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE));
+ Assert.assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE));
+ Assert.assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE));
}
+ @Test
public void testExecutePermissionsCheck(){
- assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL));
- assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE));
- assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE));
- assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE));
- assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE));
-
- assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ));
- assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE));
- assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE));
+ Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL));
+ Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE));
+ Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE));
+ Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE));
+ Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE));
+
+ Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ));
+ Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE));
+ Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE));
+
+ }
+ @Test
+ public void testGetTableSchemaWithPtnColsApi() throws IOException {
+ // Check the schema of a table with one field & no partition keys.
+ StorageDescriptor sd = new StorageDescriptor(
+ Lists.newArrayList(new FieldSchema("username", Constants.STRING_TYPE_NAME, null)),
+ "location", "org.apache.hadoop.mapred.TextInputFormat",
+ "org.apache.hadoop.mapred.TextOutputFormat", false, -1, new SerDeInfo(),
+ new ArrayList<String>(), new ArrayList<Order>(), new HashMap<String, String>());
+ Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
+ sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+ "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+
+ List<HCatFieldSchema> expectedHCatSchema =
+ Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null));
+
+ Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+ HCatUtil.getTableSchemaWithPtnCols(table));
+
+ // Add a partition key & ensure its reflected in the schema.
+ List<FieldSchema> partitionKeys =
+ Lists.newArrayList(new FieldSchema("dt", Constants.STRING_TYPE_NAME, null));
+ table.setPartitionKeys(partitionKeys);
+ expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null));
+ Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+ HCatUtil.getTableSchemaWithPtnCols(table));
}
+ /**
+ * Hive represents tables in two ways:
+ * <ul>
+ * <li>org.apache.hadoop.hive.metastore.api.Table - exactly whats stored in the metastore</li>
+ * <li>org.apache.hadoop.hive.ql.metadata.Table - adds business logic over api.Table</li>
+ * </ul>
+ * Here we check SerDe-reported fields are included in the table schema.
+ */
+ @Test
+ public void testGetTableSchemaWithPtnColsSerDeReportedFields() throws IOException {
+ Map<String, String> parameters = Maps.newHashMap();
+ parameters.put(Constants.SERIALIZATION_CLASS,
+ "org.apache.hadoop.hive.serde2.thrift.test.IntString");
+ parameters.put(Constants.SERIALIZATION_FORMAT, "org.apache.thrift.protocol.TBinaryProtocol");
+
+ SerDeInfo serDeInfo = new SerDeInfo(null,
+ "org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer", parameters);
+
+ // StorageDescriptor has an empty list of fields - SerDe will report them.
+ StorageDescriptor sd = new StorageDescriptor(new ArrayList<FieldSchema>(), "location",
+ "org.apache.hadoop.mapred.TextInputFormat", "org.apache.hadoop.mapred.TextOutputFormat",
+ false, -1, serDeInfo, new ArrayList<String>(), new ArrayList<Order>(),
+ new HashMap<String, String>());
+
+ Table table = new Table("test_tblname", "test_dbname", "test_owner", 0, 0, 0,
+ sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
+ "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
+
+ List<HCatFieldSchema> expectedHCatSchema = Lists.newArrayList(
+ new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null),
+ new HCatFieldSchema("mystring", HCatFieldSchema.Type.STRING, null),
+ new HCatFieldSchema("underscore_int", HCatFieldSchema.Type.INT, null));
+
+ Assert.assertEquals(new HCatSchema(expectedHCatSchema),
+ HCatUtil.getTableSchemaWithPtnCols(table));
+ }
}