You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/03/03 20:58:28 UTC
svn commit: r1296702 - in /incubator/hcatalog/trunk: ./ conf/
src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapreduce/
src/test/org/apache/hcatalog/mapreduce/ src/test/org/apache/hcatalog/pig/
Author: gates
Date: Sat Mar 3 20:58:28 2012
New Revision: 1296702
URL: http://svn.apache.org/viewvc?rev=1296702&view=rev
Log:
HCATALOG-279 Remove remaining code mentioning isd/osd
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/conf/proto-hive-site.xml
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sat Mar 3 20:58:28 2012
@@ -33,6 +33,8 @@ Trunk (unreleased changes)
Release 0.4.0 - Unreleased
INCOMPATIBLE CHANGES
+ HCAT-279 Remove remaining code mentioning isd/osd (daijy via gates)
+
HCAT-261 Enable more DDL statements for HCat (daijy via gates)
HCAT-252 Rework HBase storage driver into HBase storage handler (rohini via toffer)
Modified: incubator/hcatalog/trunk/conf/proto-hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/conf/proto-hive-site.xml?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/conf/proto-hive-site.xml (original)
+++ incubator/hcatalog/trunk/conf/proto-hive-site.xml Sat Mar 3 20:58:28 2012
@@ -110,12 +110,6 @@
</property>
<property>
- <name>hive.metastore.partition.inherit.table.properties</name>
- <value>hcat.isd,hcat.osd</value>
- <description>HCatalog sets this property value to hcat.isd, hcat.osd so that there is no need to do alter table set file format after adding partitions to the table.</description>
-</property>
-
-<property>
<name>hive.security.authorization.enabled</name>
<value>true</value>
<description>enable or disable the hive client authorization</description>
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java Sat Mar 3 20:58:28 2012
@@ -22,12 +22,6 @@ import org.apache.hadoop.mapred.Sequence
public final class HCatConstants {
- /** The key for the input storage driver class name */
- public static final String HCAT_ISD_CLASS = "hcat.isd";
-
- /** The key for the output storage driver class name */
- public static final String HCAT_OSD_CLASS = "hcat.osd";
-
public static final String HIVE_RCFILE_IF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
public static final String HIVE_RCFILE_OF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileOutputFormat";
public static final String HCAT_RCFILE_ISD_CLASS = "org.apache.hcatalog.rcfile.RCFileInputDriver";
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java Sat Mar 3 20:58:28 2012
@@ -511,8 +511,6 @@ class FileOutputCommitterContainer exten
private Map<String, String> getStorerParameterMap(StorerInfo storer) {
Map<String, String> params = new HashMap<String, String>();
- params.put(HCatConstants.HCAT_ISD_CLASS, storer.getInputSDClass());
- params.put(HCatConstants.HCAT_OSD_CLASS, storer.getOutputSDClass());
//Copy table level hcat.* keys to the partition
for(Entry<Object, Object> entry : storer.getProperties().entrySet()) {
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Sat Mar 3 20:58:28 2012
@@ -171,8 +171,6 @@ public abstract class HCatMapReduceTest
sd.setOutputFormat(outputFormat);
Map<String, String> tableParams = new HashMap<String, String>();
- tableParams.put(HCatConstants.HCAT_ISD_CLASS, inputSD);
- tableParams.put(HCatConstants.HCAT_OSD_CLASS, outputSD);
tbl.setParameters(tableParams);
client.createTable(tbl);
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java Sat Mar 3 20:58:28 2012
@@ -121,8 +121,6 @@ public class TestHCatOutputFormat extend
tbl.setPartitionKeys(fields);
Map<String, String> tableParams = new HashMap<String, String>();
- tableParams.put(HCatConstants.HCAT_OSD_CLASS, RCFileOutputDriver.class.getName());
- tableParams.put(HCatConstants.HCAT_ISD_CLASS, "testInputClass");
tableParams.put("hcat.testarg", "testArgValue");
tbl.setParameters(tableParams);
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1296702&r1=1296701&r2=1296702&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java Sat Mar 3 20:58:28 2012
@@ -106,9 +106,7 @@ public class TestHCatStorer extends Test
public void testPartColsInData() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -146,9 +144,7 @@ public class TestHCatStorer extends Test
driver.run("drop table employee");
String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " +
- " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
@@ -192,9 +188,7 @@ public class TestHCatStorer extends Test
public void testStoreInPartiitonedTbl() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE " +
- "tblproperties('hcat.isd'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='org.apache.hcatalog.rcfile.RCFileOutputDriver') ";
+ String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -230,9 +224,7 @@ public class TestHCatStorer extends Test
public void testNoAlias() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_parted");
- String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -277,17 +269,13 @@ public class TestHCatStorer extends Test
public void testStoreMultiTables() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver') ";
+ String createTable = "create table junit_unparted(a int, b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
}
driver.run("drop table junit_unparted2");
- createTable = "create table junit_unparted2(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ createTable = "create table junit_unparted2(a int, b string) stored as RCFILE";
retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -338,9 +326,7 @@ public class TestHCatStorer extends Test
public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int, b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -380,9 +366,7 @@ public class TestHCatStorer extends Test
public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int, b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -422,9 +406,7 @@ public class TestHCatStorer extends Test
public void testEmptyStore() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int, b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -461,9 +443,7 @@ public class TestHCatStorer extends Test
public void testBagNStruct() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
String createTable = "create table junit_unparted(b string,a struct<a1:int>, arr_of_struct array<string>, " +
- "arr_of_struct2 array<struct<s1:string,s2:string>>, arr_of_struct3 array<struct<s3:string>>) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ "arr_of_struct2 array<struct<s1:string,s2:string>>, arr_of_struct3 array<struct<s3:string>>) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -499,9 +479,7 @@ public class TestHCatStorer extends Test
public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -557,9 +535,7 @@ public class TestHCatStorer extends Test
public void testStoreFuncSimple() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ String createTable = "create table junit_unparted(a int, b string) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
throw new IOException("Failed to create table.");
@@ -603,9 +579,7 @@ public class TestHCatStorer extends Test
driver.run("drop table if exists employee");
String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " +
- " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
@@ -644,9 +618,7 @@ public class TestHCatStorer extends Test
driver.run("drop table if exists employee");
String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " +
- " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {
@@ -685,9 +657,7 @@ public class TestHCatStorer extends Test
driver.run("drop table if exists employee");
String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " +
- " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE " +
- "tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
- "'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
+ " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if(retCode != 0) {