You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/02/08 05:39:13 UTC
svn commit: r1565916 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/PigServer.java src/org/apache/pig/PigWarning.java
src/org/apache/pig/impl/plan/CompilationMessageCollector.java
src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java
Author: cheolsoo
Date: Sat Feb 8 04:39:13 2014
New Revision: 1565916
URL: http://svn.apache.org/r1565916
Log:
PIG-3447: Compiler warning message dropped for CastLineageSetter and others with no enum kind (knoguchi via cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/PigWarning.java
pig/trunk/src/org/apache/pig/impl/plan/CompilationMessageCollector.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1565916&r1=1565915&r2=1565916&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Feb 8 04:39:13 2014
@@ -87,6 +87,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3447: Compiler warning message dropped for CastLineageSetter and others with no enum kind (knoguchi via cheolsoo)
+
PIG-3627: Json storage : Doesn't work in cases , where other Store Functions (like PigStorage / AvroStorage)
do work (ssvinarchukhorton via daijy)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1565916&r1=1565915&r2=1565916&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Sat Feb 8 04:39:13 2014
@@ -1731,6 +1731,9 @@ public class PigServer {
CompilationMessageCollector collector = new CompilationMessageCollector() ;
new TypeCheckingRelVisitor( lp, collector).visit();
+ new UnionOnSchemaSetter( lp ).visit();
+ new CastLineageSetter(lp, collector).visit();
+ new ScalarVariableValidator(lp).visit();
if(aggregateWarning) {
CompilationMessageCollector.logMessages(collector, MessageType.Warning, aggregateWarning, log);
} else {
@@ -1739,9 +1742,6 @@ public class PigServer {
}
}
- new UnionOnSchemaSetter( lp ).visit();
- new CastLineageSetter(lp, collector).visit();
- new ScalarVariableValidator(lp).visit();
}
private void postProcess() throws IOException {
Modified: pig/trunk/src/org/apache/pig/PigWarning.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigWarning.java?rev=1565916&r1=1565915&r2=1565916&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigWarning.java (original)
+++ pig/trunk/src/org/apache/pig/PigWarning.java Sat Feb 8 04:39:13 2014
@@ -66,6 +66,7 @@ public enum PigWarning {
REDUCER_COUNT_LOW,
NULL_COUNTER_COUNT,
DELETE_FAILED,
- PROJECTION_INVALID_RANGE
+ PROJECTION_INVALID_RANGE,
+ NO_LOAD_FUNCTION_FOR_CASTING_BYTEARRAY
;
}
Modified: pig/trunk/src/org/apache/pig/impl/plan/CompilationMessageCollector.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/plan/CompilationMessageCollector.java?rev=1565916&r1=1565915&r2=1565916&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/plan/CompilationMessageCollector.java (original)
+++ pig/trunk/src/org/apache/pig/impl/plan/CompilationMessageCollector.java Sat Feb 8 04:39:13 2014
@@ -26,11 +26,11 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.pig.PigWarning;
/***
- * This class is used for collecting all messages (error + warning) in
- * compilation process. These messages are reported back to users
+ * This class is used for collecting all messages (error + warning) in
+ * compilation process. These messages are reported back to users
* at the end of compilation.
- *
- * iterator() has to be called after CompilationMessageCollector is fully
+ *
+ * iterator() has to be called after CompilationMessageCollector is fully
* populated otherwise the state is undefined.
*/
public class CompilationMessageCollector implements Iterable<CompilationMessageCollector.Message> {
@@ -41,49 +41,57 @@ public class CompilationMessageCollector
Warning,
Info
}
-
+
+ public enum Unknown {
+ UNKNOWN_MESSAGE_KIND;
+ public String toString() {
+ return "Aggregated unknown kind messages. Please set -Daggregate.warning=false to retrieve these messages";
+ }
+ }
+
public static class Message {
private String msg = null ;
private MessageType msgType = MessageType.Unknown ;
private Enum kind = null;
-
+
public Message(String message, MessageType messageType) {
msg = message ;
msgType = messageType ;
}
-
+
public Message(String message, MessageType messageType, Enum kind) {
- this(message, messageType);
- this.kind = kind;
+ this(message, messageType);
+ this.kind = kind;
}
-
+
public String getMessage() {
return msg ;
}
-
+
public MessageType getMessageType() {
return msgType ;
}
-
+
public Enum getKind() {
- return kind;
+ return kind;
}
}
-
+
private List<Message> messageList = new ArrayList<Message>() ;
-
+
public CompilationMessageCollector() {
// nothing here
}
-
+
public void collect(String message, MessageType messageType) {
- messageList.add(new Message(message, messageType)) ;
+ messageList.add(new Message(message, messageType,
+ Unknown.UNKNOWN_MESSAGE_KIND)) ;
}
-
+
public void collect(String message, MessageType messageType, Enum kind) {
messageList.add(new Message(message, messageType, kind)) ;
}
-
+
protected boolean hasMessageType(MessageType messageType) {
Iterator<Message> iter = iterator() ;
while(iter.hasNext()) {
@@ -93,48 +101,48 @@ public class CompilationMessageCollector
}
return false ;
}
-
+
public boolean hasError() {
- return hasMessageType(MessageType.Error);
+ return hasMessageType(MessageType.Error);
}
public Iterator<Message> iterator() {
return messageList.iterator() ;
}
-
+
public boolean hasMessage() {
return messageList.size() > 0 ;
}
-
+
public int size() {
return messageList.size() ;
}
-
+
public Message get(int i) {
return messageList.get(i) ;
}
-
+
public Map<Enum, Long> getKindAggregate(MessageType messageType) {
- Map<Enum, Long> aggMap = new HashMap<Enum, Long>();
+ Map<Enum, Long> aggMap = new HashMap<Enum, Long>();
Iterator<Message> iter = iterator() ;
while(iter.hasNext()) {
- Message message = iter.next();
+ Message message = iter.next();
if (message.getMessageType() == messageType) {
- Enum kind = message.getKind();
- if(kind != null) {
- Long count = aggMap.get(kind);
- count = (count == null? 1 : ++count);
- aggMap.put(kind, count);
- }
+ Enum kind = message.getKind();
+ if(kind != null) {
+ Long count = aggMap.get(kind);
+ count = (count == null? 1 : ++count);
+ aggMap.put(kind, count);
+ }
}
- }
- return aggMap;
+ }
+ return aggMap;
}
-
+
public static void logAggregate(Map<Enum, Long> aggMap, MessageType messageType, Log log) {
long nullCounterCount = aggMap.get(PigWarning.NULL_COUNTER_COUNT)==null?0 : aggMap.get(PigWarning.NULL_COUNTER_COUNT);
if (nullCounterCount!=0 && aggMap.size()>1) // PigWarning.NULL_COUNTER_COUNT is definitely in appMap
- logMessage("Unable to retrieve hadoop counter for " + nullCounterCount +
+ logMessage("Unable to retrieve hadoop counter for " + nullCounterCount +
" jobs, the number following warnings may not be correct", messageType, log);
for(Map.Entry<Enum, Long> e: aggMap.entrySet()) {
if (e.getKey() !=PigWarning.NULL_COUNTER_COUNT)
@@ -145,52 +153,52 @@ public class CompilationMessageCollector
logMessage(message, messageType, log);
}
}
- }
+ }
}
-
- public static void logMessages(CompilationMessageCollector messageCollector,
- MessageType messageType, boolean aggregate, Log log) {
- if(aggregate) {
- Map<Enum, Long> aggMap = messageCollector.getKindAggregate(messageType);
- logAggregate(aggMap, messageType, log);
- } else {
- Iterator<Message> messageIter = messageCollector.iterator();
- while(messageIter.hasNext()) {
- Message message = messageIter.next();
- if(message.getMessageType() == messageType) {
- logMessage(message.getMessage(), messageType, log);
- }
- }
- }
+
+ public static void logMessages(CompilationMessageCollector messageCollector,
+ MessageType messageType, boolean aggregate, Log log) {
+ if(aggregate) {
+ Map<Enum, Long> aggMap = messageCollector.getKindAggregate(messageType);
+ logAggregate(aggMap, messageType, log);
+ } else {
+ Iterator<Message> messageIter = messageCollector.iterator();
+ while(messageIter.hasNext()) {
+ Message message = messageIter.next();
+ if(message.getMessageType() == messageType) {
+ logMessage(message.getMessage(), messageType, log);
+ }
+ }
+ }
}
-
+
public void logMessages(MessageType messageType, boolean aggregate, Log log) {
- logMessages(this, messageType, aggregate, log);
+ logMessages(this, messageType, aggregate, log);
}
-
+
public static void logAllMessages(CompilationMessageCollector messageCollector, Log log) {
- Iterator<Message> messageIter = messageCollector.iterator();
- while(messageIter.hasNext()) {
- Message message = messageIter.next();
- logMessage(message.getMessage(), message.getMessageType(), log);
- }
+ Iterator<Message> messageIter = messageCollector.iterator();
+ while(messageIter.hasNext()) {
+ Message message = messageIter.next();
+ logMessage(message.getMessage(), message.getMessageType(), log);
+ }
}
-
+
public void logAllMessages(Log log) {
- logAllMessages(this, log);
+ logAllMessages(this, log);
}
private static void logMessage(String messageString, MessageType messageType, Log log) {
- switch(messageType) {
- case Info:
- log.info(messageString);
- break;
- case Warning:
- log.warn(messageString);
- break;
- case Error:
- log.error(messageString);
- }
+ switch(messageType) {
+ case Info:
+ log.info(messageString);
+ break;
+ case Warning:
+ log.warn(messageString);
+ break;
+ case Error:
+ log.error(messageString);
+ }
}
-
+
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java?rev=1565916&r1=1565915&r2=1565916&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/CastLineageSetter.java Sat Feb 8 04:39:13 2014
@@ -20,6 +20,7 @@ package org.apache.pig.newplan.logical.v
import java.util.Map;
import org.apache.pig.FuncSpec;
+import org.apache.pig.PigWarning;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.plan.CompilationMessageCollector;
@@ -107,7 +108,8 @@ public class CastLineageSetter extends A
String msg = "Cannot resolve load function to use for casting from " +
DataType.findTypeName(inType) + " to " +
DataType.findTypeName(outType) + ". ";
- msgCollector.collect(msg, MessageType.Warning);
+ msgCollector.collect(msg, MessageType.Warning,
+ PigWarning.NO_LOAD_FUNCTION_FOR_CASTING_BYTEARRAY);
}else {
cast.setFuncSpec(inLoadFunc);
}