You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/08/22 23:37:21 UTC
svn commit: r1619936 [49/49] - in /hive/branches/cbo: ./ accumulo-handler/
ant/src/org/apache/hadoop/hive/ant/ bin/
common/src/java/org/apache/hadoop/hive/ant/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hadoop/hive/c...
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java Fri Aug 22 21:36:47 2014
@@ -41,13 +41,13 @@ import org.apache.hadoop.hive.serde2.typ
import org.junit.Test;
public class TestAvroObjectInspectorGenerator {
- private final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string");
- private final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int");
- private final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean");
- private final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
- private final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float");
- private final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double");
- private final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void");
+ private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string");
+ private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int");
+ private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean");
+ private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
+ private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float");
+ private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double");
+ private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void");
// These schemata are used in other tests
static public final String MAP_WITH_PRIMITIVE_VALUE_TYPE = "{\n" +
@@ -265,7 +265,7 @@ public class TestAvroObjectInspectorGene
" \"symbols\" : [\"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\"]\n" +
"}";
- Schema s = Schema.parse(nonRecordSchema);
+ Schema s = AvroSerdeUtils.getSchemaFor(nonRecordSchema);
try {
new AvroObjectInspectorGenerator(s);
fail("Should not be able to handle non-record Avro types");
@@ -311,7 +311,7 @@ public class TestAvroObjectInspectorGene
" }\n" +
" ]\n" +
"}";
- AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(Schema.parse(bunchOfPrimitives));
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(AvroSerdeUtils.getSchemaFor(bunchOfPrimitives));
String [] expectedColumnNames = {"aString", "anInt", "aBoolean", "aLong", "aFloat", "aDouble", "aNull"};
verifyColumnNames(expectedColumnNames, aoig.getColumnNames());
@@ -350,7 +350,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException {
- Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(MAP_WITH_PRIMITIVE_VALUE_TYPE);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
verifyMap(aoig, "aMap");
}
@@ -379,7 +379,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleArrays() throws SerDeException {
- Schema s = Schema.parse(ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
+ Schema s = AvroSerdeUtils.getSchemaFor(ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -398,7 +398,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleRecords() throws SerDeException {
- Schema s = Schema.parse(RECORD_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(RECORD_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -429,7 +429,7 @@ public class TestAvroObjectInspectorGene
@Test
public void canHandleUnions() throws SerDeException {
- Schema s = Schema.parse(UNION_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(UNION_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names
@@ -452,7 +452,7 @@ public class TestAvroObjectInspectorGene
@Test // Enums are one of two Avro types that Hive doesn't have any native support for.
public void canHandleEnums() throws SerDeException {
- Schema s = Schema.parse(ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(ENUM_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
// Column names - we lose the enumness of this schema
@@ -466,7 +466,7 @@ public class TestAvroObjectInspectorGene
@Test // Hive has no concept of Avro's fixed type. Fixed -> arrays of bytes
public void canHandleFixed() throws SerDeException {
- Schema s = Schema.parse(FIXED_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(FIXED_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
@@ -483,7 +483,7 @@ public class TestAvroObjectInspectorGene
@Test // Avro considers bytes primitive, Hive doesn't. Make them list of tinyint.
public void canHandleBytes() throws SerDeException {
- Schema s = Schema.parse(BYTES_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(BYTES_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
@@ -500,7 +500,7 @@ public class TestAvroObjectInspectorGene
@Test // That Union[T, NULL] is converted to just T.
public void convertsNullableTypes() throws SerDeException {
- Schema s = Schema.parse(NULLABLE_STRING_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_STRING_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
assertEquals(1, aoig.getColumnNames().size());
@@ -517,14 +517,14 @@ public class TestAvroObjectInspectorGene
@Test // That Union[T, NULL] is converted to just T, within a Map
public void convertsMapsWithNullablePrimitiveTypes() throws SerDeException {
- Schema s = Schema.parse(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
verifyMap(aoig, "aMap");
}
@Test // That Union[T, NULL] is converted to just T.
public void convertsNullableEnum() throws SerDeException {
- Schema s = Schema.parse(NULLABLE_ENUM_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_ENUM_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
assertEquals(1, aoig.getColumnNames().size());
@@ -542,10 +542,10 @@ public class TestAvroObjectInspectorGene
@Test
public void objectInspectorsAreCached() throws SerDeException {
// Verify that Hive is caching the object inspectors for us.
- Schema s = Schema.parse(KITCHEN_SINK_SCHEMA);
+ Schema s = AvroSerdeUtils.getSchemaFor(KITCHEN_SINK_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
- Schema s2 = Schema.parse(KITCHEN_SINK_SCHEMA);
+ Schema s2 = AvroSerdeUtils.getSchemaFor(KITCHEN_SINK_SCHEMA);
AvroObjectInspectorGenerator aoig2 = new AvroObjectInspectorGenerator(s2);
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java Fri Aug 22 21:36:47 2014
@@ -61,8 +61,8 @@ public class TestAvroSerde {
" ]\n" +
"}";
- static final Schema originalSchema = Schema.parse(originalSchemaString);
- static final Schema newSchema = Schema.parse(newSchemaString);
+ static final Schema originalSchema = AvroSerdeUtils.getSchemaFor(originalSchemaString);
+ static final Schema newSchema = AvroSerdeUtils.getSchemaFor(newSchemaString);
@Test
public void initializeDoesNotReuseSchemasFromConf() throws SerDeException {
@@ -81,7 +81,7 @@ public class TestAvroSerde {
// Verify that the schema now within the configuration is the one passed
// in via the properties
- assertEquals(newSchema, Schema.parse(conf.get(AVRO_SERDE_SCHEMA)));
+ assertEquals(newSchema, AvroSerdeUtils.getSchemaFor(conf.get(AVRO_SERDE_SCHEMA)));
}
@Test
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java Fri Aug 22 21:36:47 2014
@@ -58,7 +58,7 @@ public class TestAvroSerdeUtils {
"}";
private void testField(String schemaString, String fieldName, boolean shouldBeNullable) {
- Schema s = Schema.parse(schemaString);
+ Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
assertEquals(shouldBeNullable, isNullableType(s.getField(fieldName).schema()));
}
@@ -106,11 +106,11 @@ public class TestAvroSerdeUtils {
@Test
public void getTypeFromNullableTypePositiveCase() {
- Schema s = Schema.parse(NULLABLE_UNION);
+ Schema s = AvroSerdeUtils.getSchemaFor(NULLABLE_UNION);
Schema typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
- s = Schema.parse(NULLABLE_UNION2);
+ s = AvroSerdeUtils.getSchemaFor(NULLABLE_UNION2);
typeFromNullableType = getOtherTypeFromNullableType(s.getField("mayBeNull").schema());
assertEquals(Schema.Type.STRING, typeFromNullableType.getType());
}
@@ -126,7 +126,7 @@ public class TestAvroSerdeUtils {
String schema = TestAvroObjectInspectorGenerator.RECORD_SCHEMA;
Properties props = new Properties();
props.put(AvroSerdeUtils.SCHEMA_LITERAL, schema);
- Schema expected = Schema.parse(schema);
+ Schema expected = AvroSerdeUtils.getSchemaFor(schema);
assertEquals(expected, AvroSerdeUtils.determineSchemaOrThrowException(props));
}
@@ -163,7 +163,7 @@ public class TestAvroSerdeUtils {
try {
s = determineSchemaOrThrowException(props);
assertNotNull(s);
- assertEquals(Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s);
+ assertEquals(AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s);
} catch(AvroSerdeException he) {
fail("Should have parsed schema literal, not thrown exception.");
}
@@ -197,7 +197,7 @@ public class TestAvroSerdeUtils {
Schema schemaFromHDFS =
AvroSerdeUtils.getSchemaFromFS(onHDFS, miniDfs.getFileSystem().getConf());
- Schema expectedSchema = Schema.parse(schemaString);
+ Schema expectedSchema = AvroSerdeUtils.getSchemaFor(schemaString);
assertEquals(expectedSchema, schemaFromHDFS);
} finally {
if(miniDfs != null) miniDfs.shutdown();
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java Fri Aug 22 21:36:47 2014
@@ -53,7 +53,7 @@ public class TestAvroSerializer {
" \"fields\": [" +
recordValues +
" ] }";
- return Schema.parse(s);
+ return AvroSerdeUtils.getSchemaFor(s);
}
/**
@@ -231,7 +231,7 @@ public class TestAvroSerializer {
public void canSerializeEnums() throws SerDeException, IOException {
String type = "{\"type\": \"enum\", \"name\": \"enum1_values\", " +
"\"symbols\":[\"BLUE\",\"RED\",\"GREEN\"]}";
- Schema schema = Schema.parse(type);
+ Schema schema = AvroSerdeUtils.getSchemaFor(type);
String field = "{ \"name\":\"enum1\", \"type\": " + schema + " }";
for(enum1 e : enum1.values()) {
GenericEnumSymbol symbol = new GenericData.EnumSymbol(schema, e.toString());
@@ -247,7 +247,7 @@ public class TestAvroSerializer {
String type = "{\"type\": \"enum\", \"name\": \"enum1_values\",\n" +
" \"namespace\": \"org.apache.hadoop.hive\",\n" +
" \"symbols\":[\"BLUE\",\"RED\",\"GREEN\"]}";
- Schema schema = Schema.parse(type);
+ Schema schema = AvroSerdeUtils.getSchemaFor(type);
String field = "{ \"name\":\"nullableenum\", \"type\": [\"null\", " + schema + "] }";
GenericEnumSymbol symbol = new GenericData.EnumSymbol(schema, enum1.BLUE.toString());
GenericRecord r = serializeAndDeserialize(field, "nullableenum", symbol);
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java Fri Aug 22 21:36:47 2014
@@ -50,7 +50,7 @@ public class TestGenericAvroRecordWritab
@Test
public void writableContractIsImplementedCorrectly() throws IOException {
- Schema schema = Schema.parse(schemaJSON);
+ Schema schema = AvroSerdeUtils.getSchemaFor(schemaJSON);
GenericRecord gr = new GenericData.Record(schema);
gr.put("first", "The");
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java Fri Aug 22 21:36:47 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.Ser
import org.junit.Test;
public class TestSchemaReEncoder {
+
@Test
public void schemasCanAddFields() throws SerDeException {
String original = "{\n" +
@@ -56,8 +57,8 @@ public class TestSchemaReEncoder {
" }\n" +
" ]\n" +
"}";
- Schema originalSchema = Schema.parse(original);
- Schema evolvedSchema = Schema.parse(evolved);
+ Schema originalSchema = AvroSerdeUtils.getSchemaFor(original);
+ Schema evolvedSchema = AvroSerdeUtils.getSchemaFor(evolved);
GenericRecord record = new GenericData.Record(originalSchema);
record.put("text", "it is a far better thing I do, yadda, yadda");
@@ -97,8 +98,8 @@ public class TestSchemaReEncoder {
" }\n" +
" ]\n" +
"}";
- Schema originalSchema2 = Schema.parse(original2);
- Schema evolvedSchema2 = Schema.parse(evolved2);
+ Schema originalSchema2 = AvroSerdeUtils.getSchemaFor(original2);
+ Schema evolvedSchema2 = AvroSerdeUtils.getSchemaFor(evolved2);
record = new GenericData.Record(originalSchema2);
record.put("a", 19);
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestThatEvolvedSchemasActAsWeWant.java Fri Aug 22 21:36:47 2014
@@ -34,6 +34,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
public class TestThatEvolvedSchemasActAsWeWant {
+
@Test
public void resolvedSchemasShouldReturnReaderSchema() throws IOException {
// Need to verify that when reading a datum with an updated reader schema
@@ -68,7 +69,7 @@ public class TestThatEvolvedSchemasActAs
" ]\n" +
"}";
- Schema[] schemas = {Schema.parse(v0), Schema.parse(v1)};
+ Schema[] schemas = {AvroSerdeUtils.getSchemaFor(v0), AvroSerdeUtils.getSchemaFor(v1)};
// Encode a schema with v0, write out.
GenericRecord record = new GenericData.Record(schemas[0]);
Modified: hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java (original)
+++ hive/branches/cbo/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java Fri Aug 22 21:36:47 2014
@@ -86,7 +86,8 @@ public class TestTypeInfoToSchema {
"\"fields\":[" +
"{\"name\":\"testCol\"," +
"\"type\":" + specificSchema + "," +
- "\"doc\":\"\"}" +
+ "\"doc\":\"\"," +
+ "\"default\":null}" +
"]}";
}
Modified: hive/branches/cbo/serde/src/test/resources/avro-nested-struct.avsc
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/resources/avro-nested-struct.avsc?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/resources/avro-nested-struct.avsc (original)
+++ hive/branches/cbo/serde/src/test/resources/avro-nested-struct.avsc Fri Aug 22 21:36:47 2014
@@ -5,15 +5,15 @@
"doc":"struct<superfield1:string,superfield2:struct<field1:string,field2:int>>",
"fields":
[
-{"name":"superfield1","type":["null","string"],"doc":"string"},
+{"name":"superfield1","type":["null","string"],"doc":"string","default":null},
{"name":"superfield2","type":["null",{"type":"record","name":"record_0",
"doc":"struct<field1:string,field2:int>",
"fields":
[
-{"name":"field1","type":["null","string"],"doc":"string"},
-{"name":"field2","type":["null","int"],"doc":"int"}
+{"name":"field1","type":["null","string"],"doc":"string","default":null},
+{"name":"field2","type":["null","int"],"doc":"int","default":null}
]
}
],
-"doc":"struct<field1:string,field2:int>"}]
+"doc":"struct<field1:string,field2:int>","default":null}]
}
\ No newline at end of file
Modified: hive/branches/cbo/serde/src/test/resources/avro-struct.avsc
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/test/resources/avro-struct.avsc?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/test/resources/avro-struct.avsc (original)
+++ hive/branches/cbo/serde/src/test/resources/avro-struct.avsc Fri Aug 22 21:36:47 2014
@@ -6,19 +6,19 @@
field6:smallint,field7:int,field8:bigint,field9:float,field10:double,field11:boolean,
field12:decimal(4,2),field13:void>",
"fields":[
-{"name":"field1","type":["null","string"],"doc":"string"},
-{"name":"field2","type":["null","string"],"doc":"char(5)"},
-{"name":"field3","type":["null","string"],"doc":"varchar(5)"},
-{"name":"field4","type":["null","bytes"],"doc":"binary"},
-{"name":"field5","type":["null","int"],"doc":"tinyint"},
-{"name":"field6","type":["null","int"],"doc":"smallint"},
-{"name":"field7","type":["null","int"],"doc":"int"},
-{"name":"field8","type":["null","long"],"doc":"bigint"},
-{"name":"field9","type":["null","float"],"doc":"float"},
-{"name":"field10","type":["null","double"],"doc":"double"},
-{"name":"field11","type":["null","boolean"],"doc":"boolean"},
+{"name":"field1","type":["null","string"],"doc":"string","default":null},
+{"name":"field2","type":["null","string"],"doc":"char(5)","default":null},
+{"name":"field3","type":["null","string"],"doc":"varchar(5)","default":null},
+{"name":"field4","type":["null","bytes"],"doc":"binary","default":null},
+{"name":"field5","type":["null","int"],"doc":"tinyint","default":null},
+{"name":"field6","type":["null","int"],"doc":"smallint","default":null},
+{"name":"field7","type":["null","int"],"doc":"int","default":null},
+{"name":"field8","type":["null","long"],"doc":"bigint","default":null},
+{"name":"field9","type":["null","float"],"doc":"float","default":null},
+{"name":"field10","type":["null","double"],"doc":"double","default":null},
+{"name":"field11","type":["null","boolean"],"doc":"boolean","default":null},
{"name":"field12","type":["null",{"type":"bytes","logicalType":"decimal","precision":4,
-"scale":2}],"doc":"decimal(4,2)"},
-{"name":"field13","type":"null","doc":"void"}
+"scale":2}],"doc":"decimal(4,2)","default":null},
+{"name":"field13","type":"null","doc":"void","default":null}
]
}
\ No newline at end of file
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java (original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java Fri Aug 22 21:36:47 2014
@@ -377,7 +377,7 @@ public class CLIService extends Composit
LOG.trace(opHandle + ": The background operation was cancelled", e);
} catch (ExecutionException e) {
// The background operation thread was aborted
- LOG.trace(opHandle + ": The background operation was aborted", e);
+ LOG.warn(opHandle + ": The background operation was aborted", e);
} catch (InterruptedException e) {
// No op, this thread was interrupted
// In this case, the call might return sooner than long polling timeout
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java (original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java Fri Aug 22 21:36:47 2014
@@ -55,7 +55,7 @@ public interface HiveSessionBase {
* Initialize the session
* @param sessionConfMap
*/
- public void initialize(Map<String, String> sessionConfMap);
+ public void initialize(Map<String, String> sessionConfMap) throws Exception;
public SessionHandle getSessionHandle();
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Fri Aug 22 21:36:47 2014
@@ -110,7 +110,7 @@ public class HiveSessionImpl implements
}
@Override
- public void initialize(Map<String, String> sessionConfMap) {
+ public void initialize(Map<String, String> sessionConfMap) throws Exception {
//process global init file: .hiverc
processGlobalInitFile();
SessionState.setCurrentSessionState(sessionState);
@@ -168,7 +168,7 @@ public class HiveSessionImpl implements
}
}
- private void configureSession(Map<String, String> sessionConfMap) {
+ private void configureSession(Map<String, String> sessionConfMap) throws Exception {
for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
String key = entry.getKey();
if (key.startsWith("set:")) {
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Fri Aug 22 21:36:47 2014
@@ -34,7 +34,6 @@ import org.apache.hadoop.hive.ql.hooks.H
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.auth.TSetIpAddressProcessor;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.SessionHandle;
import org.apache.hive.service.cli.operation.OperationManager;
@@ -131,15 +130,20 @@ public class SessionManager extends Comp
}
session.setSessionManager(this);
session.setOperationManager(operationManager);
- session.initialize(sessionConf);
- session.open();
- handleToSession.put(session.getSessionHandle(), session);
-
+ try {
+ session.initialize(sessionConf);
+ session.open();
+ } catch (Exception e) {
+ throw new HiveSQLException("Failed to open new session", e);
+ }
try {
executeSessionHooks(session);
} catch (Exception e) {
throw new HiveSQLException("Failed to execute session hooks", e);
}
+
+ handleToSession.put(session.getSessionHandle(), session);
+
return session.getSessionHandle();
}
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Fri Aug 22 21:36:47 2014
@@ -95,10 +95,14 @@ public class ThriftHttpCLIService extend
sslContextFactory.setKeyStorePassword(keyStorePassword);
connector = new SslSelectChannelConnector(sslContextFactory);
}
-
+
connector.setPort(portNum);
// Linux:yes, Windows:no
connector.setReuseAddress(!Shell.WINDOWS);
+
+ int maxIdleTime = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME);
+ connector.setMaxIdleTime(maxIdleTime);
+
httpServer.addConnector(connector);
hiveAuthFactory = new HiveAuthFactory(hiveConf);
Modified: hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java (original)
+++ hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java Fri Aug 22 21:36:47 2014
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.shims;
-import java.lang.IllegalArgumentException;
import java.util.HashMap;
import java.util.Map;
@@ -33,6 +32,7 @@ public abstract class ShimLoader {
private static HadoopShims hadoopShims;
private static JettyShims jettyShims;
private static AppenderSkeleton eventCounter;
+ private static HadoopThriftAuthBridge hadoopThriftAuthBridge;
/**
* The names of the classes for shimming Hadoop for each major version.
@@ -72,6 +72,22 @@ public abstract class ShimLoader {
}
/**
+ * The names of the classes for shimming {@link HadoopThriftAuthBridge}
+ */
+ private static final HashMap<String, String> HADOOP_THRIFT_AUTH_BRIDGE_CLASSES =
+ new HashMap<String, String>();
+
+ static {
+ HADOOP_THRIFT_AUTH_BRIDGE_CLASSES.put("0.20",
+ "org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge");
+ HADOOP_THRIFT_AUTH_BRIDGE_CLASSES.put("0.20S",
+ "org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge20S");
+ HADOOP_THRIFT_AUTH_BRIDGE_CLASSES.put("0.23",
+ "org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge23");
+ }
+
+
+ /**
* Factory method to get an instance of HadoopShims based on the
* version of Hadoop on the classpath.
*/
@@ -101,13 +117,12 @@ public abstract class ShimLoader {
}
public static synchronized HadoopThriftAuthBridge getHadoopThriftAuthBridge() {
- if (getHadoopShims().isSecureShimImpl()) {
- return createShim("org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge20S",
- HadoopThriftAuthBridge.class);
- } else {
- return new HadoopThriftAuthBridge();
- }
- }
+ if (hadoopThriftAuthBridge == null) {
+ hadoopThriftAuthBridge = loadShims(HADOOP_THRIFT_AUTH_BRIDGE_CLASSES,
+ HadoopThriftAuthBridge.class);
+ }
+ return hadoopThriftAuthBridge;
+ }
private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) {
String vers = getMajorVersion();
@@ -115,13 +130,12 @@ public abstract class ShimLoader {
return createShim(className, xface);
}
- private static <T> T createShim(String className, Class<T> xface) {
+ private static <T> T createShim(String className, Class<T> xface) {
try {
Class<?> clazz = Class.forName(className);
return xface.cast(clazz.newInstance());
} catch (Exception e) {
- throw new RuntimeException("Could not load shims in class " +
- className, e);
+ throw new RuntimeException("Could not load shims in class " + className, e);
}
}
Modified: hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java?rev=1619936&r1=1619935&r2=1619936&view=diff
==============================================================================
--- hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java (original)
+++ hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java Fri Aug 22 21:36:47 2014
@@ -47,6 +47,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class TestParser {
+ private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
+ .trimResults().omitEmptyStrings();
private final Context context;
private final String testCasePropertyName;
@@ -61,18 +63,17 @@ public class TestParser {
this.logger = logger;
}
private List<TestBatch> parseTests() {
- Splitter splitter = Splitter.on(" ").trimResults().omitEmptyStrings();
Context unitContext = new Context(context.getSubProperties(
Joiner.on(".").join("unitTests", "")));
- Set<String> excluded = Sets.newHashSet(splitter.split(unitContext.getString("exclude", "")));
- Set<String> isolated = Sets.newHashSet(splitter.split(unitContext.getString("isolate", "")));
- Set<String> included = Sets.newHashSet(splitter.split(unitContext.getString("include", "")));
+ Set<String> excluded = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("exclude", "")));
+ Set<String> isolated = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("isolate", "")));
+ Set<String> included = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("include", "")));
if(!included.isEmpty() && !excluded.isEmpty()) {
throw new IllegalArgumentException(String.format("Included and excluded mutally exclusive." +
" Included = %s, excluded = %s", included.toString(), excluded.toString()));
}
List<File> unitTestsDirs = Lists.newArrayList();
- for(String unitTestDir : Splitter.on(" ").omitEmptyStrings()
+ for(String unitTestDir : TEST_SPLITTER
.split(checkNotNull(unitContext.getString("directories"), "directories"))) {
File unitTestParent = new File(sourceDirectory, unitTestDir);
if(unitTestParent.isDirectory()) {
@@ -111,7 +112,6 @@ public class TestParser {
private List<QFileTestBatch> parseQFileTests() {
Map<String, Properties> properties = parseQTestProperties();
- Splitter splitter = Splitter.on(" ").trimResults().omitEmptyStrings();
List<QFileTestBatch> result = Lists.newArrayList();
for(String alias : context.getString("qFileTests", "").split(" ")) {
Context testContext = new Context(context.getSubProperties(
@@ -122,20 +122,20 @@ public class TestParser {
File directory = new File(sourceDirectory,
checkNotNull(testContext.getString("directory"), "directory").trim());
Set<String> excludedTests = Sets.newHashSet();
- for(String excludedTestGroup : splitter.split(testContext.getString("exclude", ""))) {
+ for(String excludedTestGroup : TEST_SPLITTER.split(testContext.getString("exclude", ""))) {
excludedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", excludedTestGroup), "").trim().split(" ")));
expandTestProperties(excludedTests, properties);
}
Set<String> isolatedTests = Sets.newHashSet();
- for(String ioslatedTestGroup : splitter.split(testContext.getString("isolate", ""))) {
+ for(String ioslatedTestGroup : TEST_SPLITTER.split(testContext.getString("isolate", ""))) {
isolatedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", ioslatedTestGroup), "").trim().split(" ")));
expandTestProperties(isolatedTests, properties);
}
Set<String> includedTests = Sets.newHashSet();
- for(String includedTestGroup : splitter.split(testContext.getString("include", ""))) {
+ for(String includedTestGroup : TEST_SPLITTER.split(testContext.getString("include", ""))) {
includedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", includedTestGroup), "").trim().split(" ")));
expandTestProperties(includedTests, properties);
@@ -249,7 +249,7 @@ public class TestParser {
logger.warn("No properties found in file: " + propName + " for property: " + propValue);
throw new IllegalArgumentException("No propertifies found in file: " + propName + " for property: " + propValue);
}
- Iterable<String> splits = Splitter.on(',').trimResults().omitEmptyStrings().split(result);
+ Iterable<String> splits = TEST_SPLITTER.split(result);
for (String split : splits) {
toAdd.add(split);
}