You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/15 21:13:42 UTC
svn commit: r656795 [1/3] - in /incubator/pig/branches/types: ./
src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/schema/
src/org/apache/pig/impl/logicalLayer/validators/
src/org/apache/pig/impl/plan/ test/org/apache/pig/test/
Author: gates
Date: Thu May 15 12:13:41 2008
New Revision: 656795
URL: http://svn.apache.org/viewvc?rev=656795&view=rev
Log:
PIG-143 Latest set of Pi's changes to the type checker.
Modified:
incubator/pig/branches/types/build.xml
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingValidator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/plan/PlanValidator.java
incubator/pig/branches/types/src/org/apache/pig/impl/plan/PlanWalker.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu May 15 12:13:41 2008
@@ -235,7 +235,12 @@
<path refid="classpath"/>
</classpath>
<formatter type="${test.junit.output.format}" />
- <batchtest fork="yes" todir="${test.log.dir}">
+
+ <batchtest fork="yes" todir="${test.log.dir}" if="testcase">
+ <fileset dir="test" includes="**/${testcase}.java"/>
+ </batchtest>
+
+ <batchtest fork="yes" todir="${test.log.dir}" unless="testcase">
<fileset dir="test">
<include name="**/TestBuiltin.java" />
<include name="**/TestOperatorPlan.java" />
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java Thu May 15 12:13:41 2008
@@ -66,7 +66,11 @@
return mSchema;
}
- public abstract Schema.FieldSchema getFieldSchema() throws FrontendException;
+ // Default implementation just get type info from mType
+ public Schema.FieldSchema getFieldSchema() throws FrontendException {
+ Schema.FieldSchema fs = new Schema.FieldSchema(null, mType) ;
+ return fs ;
+ }
/**
* Set the output schema for this operator. If a schema already exists, an
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java Thu May 15 12:13:41 2008
@@ -72,4 +72,10 @@
return false;
}
+ // This allows us to assign a constant to an alias
+ @Override
+ public boolean supportsMultipleOutputs() {
+ return true;
+ }
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java Thu May 15 12:13:41 2008
@@ -25,6 +25,7 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.data.DataType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -95,4 +96,8 @@
v.visit(this);
}
+ @Override
+ public byte getType() {
+ return DataType.BAG ;
+ }
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java Thu May 15 12:13:41 2008
@@ -22,6 +22,7 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.data.DataType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -94,4 +95,9 @@
v.visit(this);
}
+ @Override
+ public byte getType() {
+ return DataType.BAG ;
+ }
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java Thu May 15 12:13:41 2008
@@ -18,18 +18,14 @@
package org.apache.pig.impl.logicalLayer;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
import java.net.URL;
-import org.apache.pig.LoadFunc;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,6 +35,7 @@
private FileSpec mInputFileSpec;
private LoadFunc mLoadFunc;
private URL mSchemaFile;
+ private Schema mEnforcedSchema = null ;
private static Log log = LogFactory.getLog(LOLoad.class);
/**
@@ -93,6 +90,12 @@
try {
//DEBUG
//System.out.println("Schema file: " + mSchema);
+
+ if (mEnforcedSchema != null) {
+ mSchema = mEnforcedSchema ;
+ return mSchema ;
+ }
+
if(null != mSchemaFile) {
mSchema = mLoadFunc.determineSchema(mSchemaFile);
}
@@ -121,4 +124,17 @@
public void visit(LOVisitor v) throws VisitorException {
v.visit(this);
}
+
+ public Schema getEnforcedSchema() {
+ return mEnforcedSchema;
+ }
+
+ public void setEnforcedSchema(Schema enforcedSchema) {
+ this.mEnforcedSchema = enforcedSchema;
+ }
+
+ @Override
+ public byte getType() {
+ return DataType.BAG ;
+ }
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java Thu May 15 12:13:41 2008
@@ -59,4 +59,13 @@
public String name() {
return "Negative " + mKey.scope + "-" + mKey.id;
}
+
+ // Return type is the same as input
+ // This might not be a number type which is wrong
+ // but the post-parsing logic will eventually detect it.
+ @Override
+ public byte getType() {
+ return mOperand.getType() ;
+ }
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java Thu May 15 12:13:41 2008
@@ -228,6 +228,7 @@
} else {
mFieldSchema = new Schema.FieldSchema(null, DataType.BYTEARRAY);
}
+ mType = mFieldSchema.type ;
}
mIsFieldSchemaComputed = true;
return mFieldSchema;
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java Thu May 15 12:13:41 2008
@@ -60,6 +60,10 @@
return mOperand;
}
+ public void setOperand(ExpressionOperator op) {
+ mOperand = op ;
+ }
+
public String getRegexp() {
return mRegexp;
}
@@ -89,4 +93,9 @@
v.visit(this);
}
+ @Override
+ public byte getType() {
+ return DataType.BOOLEAN ;
+ }
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java Thu May 15 12:13:41 2008
@@ -84,7 +84,7 @@
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed && (null == mSchema)) {
// get our parent's schema
- Collection<LogicalOperator> s = mPlan.getSuccessors(this);
+ Collection<LogicalOperator> s = mPlan.getPredecessors(this);
try {
LogicalOperator op = s.iterator().next();
if (null == op) {
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java Thu May 15 12:13:41 2008
@@ -287,6 +287,22 @@
}
+ protected void visit(LOUnion u) throws VisitorException {
+
+ }
+
+ protected void visit(LOSplitOutput sop) throws VisitorException {
+
+ }
+
+ protected void visit(LODistinct dt) throws VisitorException {
+
+ }
+
+ protected void visit(LOCross cs) throws VisitorException {
+
+ }
+
protected void visit(LOProject project) throws VisitorException {
// Visit the operand of the project as long as the sentinel is false
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Thu May 15 12:13:41 2008
@@ -215,7 +215,7 @@
*/
public abstract void visit(LOVisitor v) throws VisitorException;
- /*
+ /*
public boolean isFlatten() {
return mIsFlatten;
}
@@ -223,6 +223,8 @@
public void setFlatten(boolean b) {
mIsFlatten = b;
}
- */
-
+ */
+ public LogicalPlan getPlan() {
+ return mPlan ;
+ }
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java Thu May 15 12:13:41 2008
@@ -31,7 +31,7 @@
*/
public abstract class UnaryExpressionOperator extends ExpressionOperator {
private static final long serialVersionUID = 2L;
- private ExpressionOperator mOperand; // operand
+ protected ExpressionOperator mOperand; // operand
private static Log log = LogFactory.getLog(UnaryExpressionOperator.class);
/**
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Thu May 15 12:13:41 2008
@@ -107,23 +107,33 @@
type = t;
}
+ /***
+ * Two field schemas are equal if types and schemas
+ * are equal in all levels.
+ *
+ * In order to relax alias equivalent requirement,
+ * instead use equals(FieldSchema fschema,
+ FieldSchema fother,
+ boolean relaxInner,
+ boolean relaxAlias)
+ */
+
@Override
public boolean equals(Object other) {
- if (!(other instanceof FieldSchema))
- return false;
- FieldSchema fs = (FieldSchema) other;
- // Fields can have different names and still be equal. But
- // types and schemas (if they're a tuple) must match.
- if (type != fs.type)
- return false;
- if (schema != fs.schema)
- return false;
+ if (!(other instanceof FieldSchema)) return false;
+ FieldSchema otherfs = (FieldSchema)other;
- return true;
+ return FieldSchema.equals(this, otherfs, false, false) ;
+ }
+
+
+ @Override
+ public int hashCode() {
+ return (this.type * 17)
+ + ( (schema==null? 0:schema.hashCode()) * 23 )
+ + ( (alias==null? 0:alias.hashCode()) * 29 ) ;
}
- // TODO Need to add hashcode.
-
/***
* Compare two field schema for equality
* @param fschema
@@ -132,35 +142,58 @@
* @param relaxAlias If true, we don't check aliases
* @return
*/
- public static boolean equals(FieldSchema fschema,
- FieldSchema fother,
+ public static boolean equals(FieldSchema fschema,
+ FieldSchema fother,
boolean relaxInner,
boolean relaxAlias) {
- if (fschema == null) {
+ if (fschema == null) {
return false ;
}
-
+
if (fother == null) {
return false ;
}
-
+
if (fschema.type != fother.type) {
return false ;
}
-
+
if ( (!relaxAlias) && (fschema.alias != fother.alias) ) {
return false ;
}
-
- if ( (!relaxInner) && (fschema.type == DataType.TUPLE) ) {
- // compare recursively using schema
- if (!Schema.equals(fschema.schema, fother.schema, false, relaxAlias)) {
- return false ;
- }
+
+ if ( (!relaxInner) && (DataType.isSchemaType(fschema.type))) {
+ // Don't do the comparison if both embedded schemas are
+ // null. That will cause Schema.equals to return false,
+ // even though we want to view that as true.
+ if (!(fschema.schema == null && fother.schema == null)) {
+ // compare recursively using schema
+ if (!Schema.equals(fschema.schema, fother.schema, false, relaxAlias)) {
+ return false ;
+ }
+ }
}
-
+
return true ;
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ if (alias != null) {
+ sb.append(alias);
+ sb.append(": ");
+ }
+ sb.append(DataType.findTypeName(type));
+
+ if (schema != null) {
+ sb.append("(");
+ sb.append(schema.toString());
+ sb.append(")");
+ }
+ return sb.toString();
+ }
+
}
private List<FieldSchema> mFields;
@@ -175,18 +208,17 @@
}
/**
- * @param fields
- * List of field schemas that describes the fields.
+ * @param fields List of field schemas that describes the fields.
*/
public Schema(List<FieldSchema> fields) {
mFields = fields;
mAliases = new HashMap<String, FieldSchema>(fields.size());
mFieldSchemas = new MultiMap<FieldSchema, String>();
- for (FieldSchema fs : fields) {
+ for (FieldSchema fs : fields) {
if (fs.alias != null) {
mAliases.put(fs.alias, fs);
if(null != fs) {
- mFieldSchemas.put(fs, fs.alias);
+ mFieldSchemas.put(fs, fs.alias);
}
}
}
@@ -194,9 +226,7 @@
/**
* Create a schema with only one field.
- *
- * @param fieldSchema
- * field to put in this schema.
+ * @param fieldSchema field to put in this schema.
*/
public Schema(FieldSchema fieldSchema) {
mFields = new ArrayList<FieldSchema>(1);
@@ -213,9 +243,7 @@
/**
* Given an alias name, find the associated FieldSchema.
- *
- * @param alias
- * Alias to look up.
+ * @param alias Alias to look up.
* @return FieldSchema, or null if no such alias is in this tuple.
*/
public FieldSchema getField(String alias) {
@@ -224,7 +252,7 @@
/**
* Given a field number, find the associated FieldSchema.
- *
+ *
* @param fieldNum
* Field number to look up.
* @return FieldSchema for this field.
@@ -243,7 +271,7 @@
/**
* Find the number of fields in the schema.
- *
+ *
* @return number of fields.
*/
public int size() {
@@ -251,26 +279,20 @@
}
/**
- * Reconcile this schema with another schema. The schema being reconciled
- * with should have the same number of columns. The use case is where a
- * schema already exists but may not have alias and or type information. If
+ * Reconcile this schema with another schema. The schema being reconciled
+ * with should have the same number of columns. The use case is where a
+ * schema already exists but may not have alias and or type information. If
* an alias exists in this schema and a new one is given, then the new one
- * will be used. Similarly with types, though this needs to be used
+ * will be used. Similarly with types, though this needs to be used
* carefully, as types should not be lightly changed.
- *
- * @param other
- * Schema to reconcile with.
- * @throws ParseException
- * if this cannot be reconciled.
+ * @param other Schema to reconcile with.
+ * @throws ParseException if this cannot be reconciled.
*/
public void reconcile(Schema other) throws ParseException {
if (other.size() != size()) {
- log.debug("Cannot reconcile schemas with different "
- + "sizes. This schema has size " + size()
- + " other has size " + "of " + other.size());
throw new ParseException("Cannot reconcile schemas with different "
- + "sizes. This schema has size " + size()
- + " other has size " + "of " + other.size());
+ + "sizes. This schema has size " + size() + " other has size "
+ + "of " + other.size());
}
Iterator<FieldSchema> i = other.mFields.iterator();
@@ -309,29 +331,54 @@
ourFs.schema = otherFs.schema;
log.debug("Setting schema to: " + otherFs.schema);
}
+
}
}
+ /***
+ * For two schemas to be equal, they have to be deeply equal.
+ * Use Schema.equals(Schema schema,
+ Schema other,
+ boolean relaxInner,
+ boolean relaxAlias)
+ if relaxation of aliases is a requirement.
+ */
@Override
public boolean equals(Object other) {
- if (!(other instanceof Schema))
- return false;
+ if (!(other instanceof Schema)) return false;
+
+ Schema s = (Schema)other;
+ return Schema.equals(this, s, false, false) ;
- Schema s = (Schema) other;
+ }
- if (s.size() != size())
- return false;
+ static int[] primeList = { 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37,
+ 41, 43, 47, 53, 59, 61, 67, 71, 73, 79,
+ 83, 89, 97, 101, 103, 107, 109, 1133} ;
- Iterator<FieldSchema> i = mFields.iterator();
- Iterator<FieldSchema> j = s.mFields.iterator();
- while (i.hasNext()) {
- if (!(i.next().equals(j.next())))
- return false;
+ @Override
+ public int hashCode() {
+ int idx = 0 ;
+ int hashCode = 0 ;
+ for(FieldSchema fs: this.mFields) {
+ hashCode += fs.hashCode() * (primeList[idx % primeList.length]) ;
+ idx++ ;
}
- return true;
+ return hashCode ;
}
- // TODO add hashCode()
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("(");
+ boolean first = true;
+ for (FieldSchema fs : mFields) {
+ if (first) first = false;
+ else sb.append(", ");
+ sb.append(fs.toString());
+ }
+ sb.append(")");
+ return sb.toString();
+ }
public void add(FieldSchema f) {
mFields.add(f);
@@ -339,10 +386,10 @@
mAliases.put(f.alias, f);
}
}
-
+
/**
* Given an alias, find the associated position of the field schema.
- *
+ *
* @param alias
* alias of the FieldSchema.
* @return position of the FieldSchema.
@@ -354,7 +401,7 @@
if (null == fs) {
return -1;
}
-
+
log.debug("fs: " + fs);
int index = -1;
for(int i = 0; i < mFields.size(); ++i) {
@@ -386,7 +433,7 @@
log.debug("Schema Alias: " + alias);
}
}
-
+
public List<FieldSchema> getFields() {
return mFields;
}
@@ -398,48 +445,48 @@
* @param relaxInner if true, inner schemas will not be checked
* @return
*/
- public static boolean equals(Schema schema,
- Schema other,
+ public static boolean equals(Schema schema,
+ Schema other,
boolean relaxInner,
boolean relaxAlias) {
if (schema == null) {
return false ;
}
-
+
if (other == null) {
return false ;
}
-
+
if (schema.size() != other.size()) return false;
Iterator<FieldSchema> i = schema.mFields.iterator();
Iterator<FieldSchema> j = other.mFields.iterator();
-
+
while (i.hasNext()) {
-
+
FieldSchema myFs = i.next() ;
FieldSchema otherFs = j.next() ;
-
+
if ( (!relaxAlias) && (myFs.alias != otherFs.alias) ) {
return false ;
}
-
+
if (myFs.type != otherFs.type) {
return false ;
}
-
+
if (!relaxInner) {
// Compare recursively using field schema
if (!FieldSchema.equals(myFs, otherFs, false, relaxAlias)) {
return false ;
- }
+ }
}
-
+
}
return true;
}
-
-
+
+
/***
* Merge this schema with the other schema
* @param other the other schema to be merged with
@@ -450,70 +497,70 @@
public Schema merge(Schema other, boolean otherTakesAliasPrecedence) {
return mergeSchema(this, other, otherTakesAliasPrecedence) ;
}
-
+
/***
- * Recursively merge two schemas
+ * Recursively merge two schemas
* @param schema the initial schema
* @param other the other schema to be merged with
* @param otherTakesAliasPrecedence true if aliases from the other
* schema take precedence
* @return the merged schema, null if they are not compatible
*/
- private Schema mergeSchema(Schema schema, Schema other,
+ private Schema mergeSchema(Schema schema, Schema other,
boolean otherTakesAliasPrecedence) {
-
+
if (other == null) {
return null ;
}
-
+
if (schema.size() != other.size()) {
return null ;
}
-
+
List<FieldSchema> outputList = new ArrayList<FieldSchema>() ;
-
+
Iterator<FieldSchema> mylist = schema.mFields.iterator() ;
Iterator<FieldSchema> otherlist = other.mFields.iterator() ;
-
+
while (mylist.hasNext()) {
-
+
FieldSchema myFs = mylist.next() ;
FieldSchema otherFs = otherlist.next() ;
-
+
byte mergedType = mergeType(myFs.type, otherFs.type) ;
// if the types cannot be merged, the schemas cannot be merged
if (mergedType == DataType.ERROR) {
return null ;
}
-
- String mergedAlias = mergeAlias(myFs.alias,
- otherFs.alias,
+
+ String mergedAlias = mergeAlias(myFs.alias,
+ otherFs.alias,
otherTakesAliasPrecedence) ;
-
+
FieldSchema mergedFs = null ;
if (!DataType.isSchemaType(mergedType)) {
- // just normal merge
+ // just normal merge
mergedFs = new FieldSchema(mergedAlias, mergedType) ;
}
else {
// merge inner tuple because both sides are tuples
- Schema mergedSubSchema = mergeSchema(myFs.schema,
+ Schema mergedSubSchema = mergeSchema(myFs.schema,
otherFs.schema,
otherTakesAliasPrecedence) ;
// return null if they cannot be merged
if (mergedSubSchema == null) {
return null ;
}
-
+
mergedFs = new FieldSchema(mergedAlias, mergedSubSchema) ;
-
+
}
outputList.add(mergedFs) ;
}
-
+
return new Schema(outputList) ;
}
-
+
/***
* Merge two aliases. If one of aliases is null, return the other.
* Otherwise check the precedence condition
@@ -537,7 +584,7 @@
return alias ;
}
}
-
+
/***
* Merge types if possible
* @param type1
@@ -550,33 +597,37 @@
(!DataType.isUsableType(type2)) ) {
return DataType.ERROR ;
}
-
+
// Same type is OK
if (type1==type2) {
return type1 ;
}
-
+
// Both are number so we return the bigger type
if ( (DataType.isNumberType(type1)) &&
(DataType.isNumberType(type2)) ) {
return type1>type2 ? type1:type2 ;
}
-
+
// One is bytearray and the other is (number or chararray)
if ( (type1 == DataType.BYTEARRAY) &&
( (type2 == DataType.CHARARRAY) || (DataType.isNumberType(type2)) )
) {
return type2 ;
}
-
+
if ( (type2 == DataType.BYTEARRAY) &&
( (type1 == DataType.CHARARRAY) || (DataType.isNumberType(type1)) )
) {
return type1 ;
}
-
+
// else return just ERROR
return DataType.ERROR ;
}
+
}
+
+
+
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingValidator.java?rev=656795&r1=656794&r2=656795&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingValidator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingValidator.java Thu May 15 12:13:41 2008
@@ -19,7 +19,8 @@
throws PlanValidationException {
// The first msgCollector is used in visitor
// The second msgCollector is used by the validator
- super.validate(new TypeCheckingVisitor(plan, msgCollector),
- msgCollector) ;
+ super.validateSkipCollectException(new TypeCheckingVisitor(plan, msgCollector),
+ msgCollector) ;
+
}
}