You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ha...@apache.org on 2011/06/28 16:35:13 UTC
svn commit: r1140716 - in /incubator/hcatalog/trunk: ./
src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/listener/
src/test/org/apache/hcatalog/cli/ src/test/org/apache/hcatalog/listener/
Author: hashutosh
Date: Tue Jun 28 16:35:12 2011
New Revision: 1140716
URL: http://svn.apache.org/viewvc?rev=1140716&view=rev
Log:
HCATALOG-52: No message is sent on message bus in case partition keys are uppercase
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/listener/NotificationListener.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/listener/TestNotificationListener.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1140716&r1=1140715&r2=1140716&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue Jun 28 16:35:12 2011
@@ -45,6 +45,9 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+
+ HCAT-52. No message is sent on message bus in case partition keys are uppercase (hashutosh)
+
HCAT-45. HCatalog release tar ball source code has problems (gates)
HCAT-41. Changes to hcat.sh in HCATALOG-20 broke the -e option (gates)
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java?rev=1140716&r1=1140715&r2=1140716&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java Tue Jun 28 16:35:12 2011
@@ -73,7 +73,7 @@ public final class HCatConstants {
public static final String HCAT_MSGBUS_TOPIC_PREFIX = "hcat.msgbus.topic.prefix";
// Message Bus related properties.
- public static final String HCAT_DEFAULT_TOPIC_PREFIX = "HCAT";
+ public static final String HCAT_DEFAULT_TOPIC_PREFIX = "hcat";
public static final String HCAT_EVENT = "HCAT_EVENT";
public static final String HCAT_ADD_PARTITION_EVENT = "HCAT_ADD_PARTITION";
public static final String HCAT_DROP_PARTITION_EVENT = "HCAT_DROP_PARTITION";
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/listener/NotificationListener.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/listener/NotificationListener.java?rev=1140716&r1=1140715&r2=1140716&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/listener/NotificationListener.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/listener/NotificationListener.java Tue Jun 28 16:35:12 2011
@@ -33,7 +33,6 @@ import javax.jms.JMSException;
import javax.jms.MapMessage;
import javax.jms.Message;
import javax.jms.MessageProducer;
-import javax.jms.ObjectMessage;
import javax.jms.Session;
import javax.naming.Context;
import javax.naming.InitialContext;
@@ -157,23 +156,29 @@ public class NotificationListener extend
// by listening on a topic named "HCAT" and message selector string
// as "HCAT_EVENT = HCAT_ADD_TABLE"
if(tableEvent.getStatus()){
- if(tableEvent.getStatus()){
- Table tbl = tableEvent.getTable();
- Table newTbl = tbl.deepCopy();
- HMSHandler handler = tableEvent.getHandler();
- HiveConf conf = handler.getHiveConf();
+ Table tbl = tableEvent.getTable();
+ HMSHandler handler = tableEvent.getHandler();
+ HiveConf conf = handler.getHiveConf();
+ Table newTbl;
+ try {
+ newTbl = handler.get_table(tbl.getDbName(), tbl.getTableName()).deepCopy();
newTbl.getParameters().put(HCatConstants.HCAT_MSGBUS_TOPIC_NAME,
- getTopicPrefix(conf) + "." + tbl.getDbName() +"." + tbl.getTableName());
- try {
- handler.alter_table(tbl.getDbName(), tbl.getTableName(), newTbl);
- } catch (InvalidOperationException e) {
- throw new MetaException(e.toString());
- }
- send(tableEvent.getTable(),getTopicPrefix(conf)+ "."+ tbl.getDbName(), HCatConstants.HCAT_ADD_TABLE_EVENT);
+ getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase()
+ +"." + newTbl.getTableName().toLowerCase());
+ handler.alter_table(newTbl.getDbName(), newTbl.getTableName(), newTbl);
+ } catch (InvalidOperationException e) {
+ MetaException me = new MetaException(e.toString());
+ me.initCause(e);
+ throw me;
+ } catch (NoSuchObjectException e) {
+ MetaException me = new MetaException(e.toString());
+ me.initCause(e);
+ throw me;
}
- }
+ send(newTbl,getTopicPrefix(conf)+ "."+ newTbl.getDbName().toLowerCase(), HCatConstants.HCAT_ADD_TABLE_EVENT);
+ }
}
-
+
private String getTopicPrefix(HiveConf conf){
return conf.get(HCatConstants.HCAT_MSGBUS_TOPIC_PREFIX,HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX);
}
@@ -194,7 +199,7 @@ public class NotificationListener extend
sd.setSortCols(new ArrayList<Order>());
sd.setParameters(new HashMap<String, String>());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
- send(table,getTopicPrefix(tableEvent.getHandler().getHiveConf())+"."+table.getDbName(), HCatConstants.HCAT_DROP_TABLE_EVENT);
+ send(table,getTopicPrefix(tableEvent.getHandler().getHiveConf())+"."+table.getDbName().toLowerCase(), HCatConstants.HCAT_DROP_TABLE_EVENT);
}
}
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java?rev=1140716&r1=1140715&r2=1140716&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java Tue Jun 28 16:35:12 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.listener.NotificationListener;
import org.apache.hcatalog.rcfile.RCFileInputDriver;
import org.apache.hcatalog.rcfile.RCFileOutputDriver;
import org.apache.thrift.TException;
@@ -57,6 +58,7 @@ public class TestSemanticAnalysis extend
@Override
protected void setUp() throws Exception {
+ System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, NotificationListener.class.getName());
HiveConf hcatConf = new HiveConf(this.getClass());
hcatConf.set(ConfVars.PREEXECHOOKS.varname, "");
hcatConf.set(ConfVars.POSTEXECHOOKS.varname, "");
@@ -64,7 +66,7 @@ public class TestSemanticAnalysis extend
HiveConf hiveConf = new HiveConf(hcatConf,this.getClass());
hiveDriver = new Driver(hiveConf);
-
+
hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName());
hcatDriver = new Driver(hcatConf);
@@ -75,6 +77,16 @@ public class TestSemanticAnalysis extend
String query;
private final String tblName = "junit_sem_analysis";
+ public void testCreateTblWithLowerCasePartNames() throws CommandNeedRetryException, MetaException, TException, NoSuchObjectException{
+ hiveDriver.run("drop table junit_sem_analysis");
+ CommandProcessorResponse resp = hiveDriver.run("create table junit_sem_analysis (a int) partitioned by (B string) stored as TEXTFILE");
+ assertEquals(resp.getResponseCode(), 0);
+ assertEquals(null, resp.getErrorMessage());
+ Table tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
+ assertEquals("Partition key name case problem", "b" , tbl.getPartitionKeys().get(0).getName());
+ hiveDriver.run("drop table junit_sem_analysis");
+ }
+
public void testAlterTblFFpart() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hiveDriver.run("drop table junit_sem_analysis");
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/listener/TestNotificationListener.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/listener/TestNotificationListener.java?rev=1140716&r1=1140715&r2=1140716&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/listener/TestNotificationListener.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/listener/TestNotificationListener.java Tue Jun 28 16:35:12 2011
@@ -130,7 +130,7 @@ public class TestNotificationListener ex
}
else if(event.equals(HCatConstants.HCAT_ADD_TABLE_EVENT)){
- assertEquals("topic://HCAT.mydb",msg.getJMSDestination().toString());
+ assertEquals("topic://hcat.mydb",msg.getJMSDestination().toString());
Table tbl = (Table)(((ObjectMessage)msg).getObject());
assertEquals("mytbl", tbl.getTableName());
assertEquals("mydb", tbl.getDbName());
@@ -138,7 +138,7 @@ public class TestNotificationListener ex
}
else if(event.equals(HCatConstants.HCAT_ADD_PARTITION_EVENT)){
- assertEquals("topic://HCAT.mydb.mytbl",msg.getJMSDestination().toString());
+ assertEquals("topic://hcat.mydb.mytbl",msg.getJMSDestination().toString());
Partition part = (Partition)(((ObjectMessage)msg).getObject());
assertEquals("mytbl", part.getTableName());
assertEquals("mydb", part.getDbName());
@@ -148,7 +148,7 @@ public class TestNotificationListener ex
}
else if(event.equals(HCatConstants.HCAT_DROP_PARTITION_EVENT)){
- assertEquals("topic://HCAT.mydb.mytbl",msg.getJMSDestination().toString());
+ assertEquals("topic://hcat.mydb.mytbl",msg.getJMSDestination().toString());
Partition part = (Partition)(((ObjectMessage)msg).getObject());
assertEquals("mytbl", part.getTableName());
assertEquals("mydb", part.getDbName());
@@ -158,7 +158,7 @@ public class TestNotificationListener ex
}
else if(event.equals(HCatConstants.HCAT_DROP_TABLE_EVENT)){
- assertEquals("topic://HCAT.mydb",msg.getJMSDestination().toString());
+ assertEquals("topic://hcat.mydb",msg.getJMSDestination().toString());
Table tbl = (Table)(((ObjectMessage)msg).getObject());
assertEquals("mytbl", tbl.getTableName());
assertEquals("mydb", tbl.getDbName());
@@ -170,7 +170,7 @@ public class TestNotificationListener ex
assertEquals("mydb", ((Database) ((ObjectMessage)msg).getObject()).getName());
}
else if (event.equals(HCatConstants.HCAT_PARTITION_DONE_EVENT)) {
- assertEquals("topic://HCAT.mydb.mytbl",msg.getJMSDestination().toString());
+ assertEquals("topic://hcat.mydb.mytbl",msg.getJMSDestination().toString());
MapMessage mapMsg = (MapMessage)msg;
assert mapMsg.getString("b").equals("2011");
} else