You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2010/03/04 23:10:46 UTC
svn commit: r919202 - in /hadoop/pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/impl/logicalLayer/schema/ test/org/apache/pig/test/
test/org/apache/pig/test/utils/
Author: pradeepkth
Date: Thu Mar 4 22:10:45 2010
New Revision: 919202
URL: http://svn.apache.org/viewvc?rev=919202&view=rev
Log:
PIG-1259: ResourceFieldSchema.setSchema should not allow a bag field without a Tuple as its only sub field (the tuple itself can have a schema with > 1 subfields) (pradeepkth)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java
hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Mar 4 22:10:45 2010
@@ -22,6 +22,10 @@
INCOMPATIBLE CHANGES
+PIG-1259: ResourceFieldSchema.setSchema should not allow a bag field without a
+Tuple as its only sub field (the tuple itself can have a schema with > 1
+subfields) (pradeepkth)
+
PIG-1265: Change LoadMetadata and StoreMetadata to use Job instead of
Configuraiton and add a cleanupOnFailure method to StoreFuncInterface
(pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java Thu Mar 4 22:10:45 2010
@@ -18,6 +18,7 @@
package org.apache.pig;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
@@ -25,6 +26,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
@@ -118,11 +120,35 @@
return schema;
}
- public ResourceFieldSchema setSchema(ResourceSchema schema) {
+ public ResourceFieldSchema setSchema(ResourceSchema schema) throws
+ IOException {
+ validateSchema(schema);
this.schema = schema;
return this;
}
+ /**
+ * @param schema
+ */
+ private void validateSchema(ResourceSchema schema) throws IOException {
+ if(type == DataType.BAG && schema != null) {
+ ResourceFieldSchema[] subFields = schema.getFields();
+ if (subFields.length == 1) {
+ if (subFields[0].type != DataType.TUPLE) {
+ throwInvalidSchemaException();
+ }
+ } else {
+ throwInvalidSchemaException();
+ }
+ }
+ }
+
+ public static void throwInvalidSchemaException() throws FrontendException {
+ int errCode = 2218;
+ throw new FrontendException("Invalid resource schema: " +
+ "bag schema must have tuple as its field", errCode, PigException.BUG);
+ }
+
@Override
public String toString() {
return getDescription(true);
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Thu Mar 4 22:10:45 2010
@@ -1623,15 +1623,13 @@
if (fs.schema.size() == 1) {
FieldSchema innerFs = fs.schema.getField(0);
if (innerFs.type != DataType.TUPLE) {
- throw new FrontendException("Invalide resource schema: " +
- "bag schema must have tuple as its field.");
+ ResourceFieldSchema.throwInvalidSchemaException();
}
if (innerFs.schema != null) { // allow partial schema
fs.schema.setTwoLevelAccessRequired(true);
}
} else {
- throw new FrontendException("Invalide resource schema: " +
- "bag schema should have exact one field.");
+ ResourceFieldSchema.throwInvalidSchemaException();
}
}
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java Thu Mar 4 22:10:45 2010
@@ -21,6 +21,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -124,9 +125,10 @@
* Test that Pig Schema is correctly created given a
* ResourceSchema and vice versa. Test also that
* TwoLevelAccess flag is set for Pig Schema when needed.
+ * @throws IOException
*/
@Test
- public void testToPigSchemaWithTwoLevelAccess() throws FrontendException {
+ public void testToPigSchemaWithTwoLevelAccess() throws IOException {
ResourceFieldSchema[] level0 =
new ResourceFieldSchema[] {
new ResourceFieldSchema()
@@ -182,9 +184,10 @@
/**
* Test invalid Resource Schema: multiple fields for a bag
+ * @throws IOException
*/
@Test(expected=FrontendException.class)
- public void testToPigSchemaWithInvalidSchema() throws FrontendException {
+ public void testToPigSchemaWithInvalidSchema() throws IOException {
ResourceFieldSchema[] level0 = new ResourceFieldSchema[] {
new ResourceFieldSchema()
.setName("fld0").setType(DataType.CHARARRAY),
@@ -201,18 +204,14 @@
new ResourceFieldSchema()
.setName("t2").setType(DataType.BAG).setSchema(rSchema0)
};
-
- ResourceSchema rSchema2 = new ResourceSchema()
- .setFields(level2);
-
- Schema.getPigSchema(rSchema2);
}
/**
* Test invalid Resource Schema: bag without tuple field
+ * @throws IOException
*/
@Test(expected=FrontendException.class)
- public void testToPigSchemaWithInvalidSchema2() throws FrontendException {
+ public void testToPigSchemaWithInvalidSchema2() throws IOException {
ResourceFieldSchema[] level0 = new ResourceFieldSchema[] {
new ResourceFieldSchema()
.setName("fld0").setType(DataType.CHARARRAY)
@@ -225,11 +224,7 @@
new ResourceFieldSchema()
.setName("t2").setType(DataType.BAG).setSchema(rSchema0)
};
-
- ResourceSchema rSchema2 = new ResourceSchema()
- .setFields(level2);
-
- Schema.getPigSchema(rSchema2);
+
}
/**
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Thu Mar 4 22:10:45 2010
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test;
+import java.io.IOException;
import java.util.Map;
import org.junit.Test;
@@ -43,7 +44,7 @@
PigStorage ps = new PigStorage();
- ResourceFieldSchema getTupleFieldSchema() {
+ ResourceFieldSchema getTupleFieldSchema() throws IOException {
ResourceFieldSchema stringfs = new ResourceFieldSchema();
stringfs.setType(DataType.CHARARRAY);
ResourceFieldSchema intfs = new ResourceFieldSchema();
@@ -58,7 +59,7 @@
return tuplefs;
}
- public ResourceFieldSchema getBagFieldSchema(){
+ public ResourceFieldSchema getBagFieldSchema() throws IOException{
ResourceFieldSchema tuplefs = getTupleFieldSchema();
ResourceSchema outBagSchema = new ResourceSchema();
Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java?rev=919202&r1=919201&r2=919202&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java Thu Mar 4 22:10:45 2010
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test.utils;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -77,7 +78,7 @@
return new DataByteArray(genRandString(r).getBytes());
}
- public static ResourceFieldSchema getSmallTupleFieldSchema(){
+ public static ResourceFieldSchema getSmallTupleFieldSchema() throws IOException{
ResourceFieldSchema stringfs = new ResourceFieldSchema();
stringfs.setType(DataType.CHARARRAY);
ResourceFieldSchema intfs = new ResourceFieldSchema();
@@ -143,7 +144,7 @@
return db;
}
- public static ResourceFieldSchema getSmallTupDataBagFieldSchema() {
+ public static ResourceFieldSchema getSmallTupDataBagFieldSchema() throws IOException {
ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();
ResourceSchema bagSchema = new ResourceSchema();
@@ -190,7 +191,7 @@
return t;
}
- public static ResourceFieldSchema getSmallBagTextTupleFieldSchema(){
+ public static ResourceFieldSchema getSmallBagTextTupleFieldSchema() throws IOException{
ResourceFieldSchema stringfs = new ResourceFieldSchema();
stringfs.setType(DataType.CHARARRAY);
@@ -262,7 +263,7 @@
return db;
}
- public static ResourceFieldSchema getFullTupTextDataBagFieldSchema(){
+ public static ResourceFieldSchema getFullTupTextDataBagFieldSchema() throws IOException{
ResourceFieldSchema tuplefs = getSmallBagTextTupleFieldSchema();
ResourceSchema outBagSchema = new ResourceSchema();
@@ -347,7 +348,7 @@
return db;
}
- public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) {
+ public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) throws IOException {
ResourceFieldSchema intfs = new ResourceFieldSchema();
intfs.setType(DataType.INTEGER);
@@ -391,7 +392,7 @@
return t;
}
- public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() {
+ public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException {
ResourceFieldSchema stringfs = new ResourceFieldSchema();
stringfs.setType(DataType.CHARARRAY);
ResourceFieldSchema intfs = new ResourceFieldSchema();