You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/01/28 06:48:10 UTC
svn commit: r1561947 [2/17] - in /hive/branches/tez: ./ ant/
ant/src/org/apache/hadoop/hive/ant/ beeline/ cli/
cli/src/java/org/apache/hadoop/hive/cli/ common/ common/src/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/h...
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java Tue Jan 28 05:48:03 2014
@@ -171,20 +171,28 @@ public class JsonSerDe implements SerDe
throw new IOException("Field name expected");
}
String fieldName = p.getText();
- int fpos;
- try {
- fpos = s.getPosition(fieldName);
- } catch (NullPointerException npe) {
+ Integer fpos = s.getPosition(fieldName);
+ if (fpos == null) {
fpos = getPositionFromHiveInternalColumnName(fieldName);
- LOG.debug("NPE finding position for field [{}] in schema [{}]", fieldName, s);
+ LOG.debug("NPE finding position for field [{}] in schema [{}],"
+ +" attempting to check if it is an internal column name like _col0", fieldName, s);
+ if (fpos == -1) {
+ skipValue(p);
+ return; // unknown field, we return. We'll continue from the next field onwards.
+ }
+ // If we get past this, then the column name did match the hive pattern for an internal
+ // column name, such as _col0, etc, so it *MUST* match the schema for the appropriate column.
+ // This means people can't use arbitrary column names such as _col0, and expect us to ignore it
+ // if we find it.
if (!fieldName.equalsIgnoreCase(getHiveInternalColumnName(fpos))) {
LOG.error("Hive internal column name {} and position "
+ "encoding {} for the column name are at odds", fieldName, fpos);
- throw npe;
- }
- if (fpos == -1) {
- return; // unknown field, we return.
+ throw new IOException("Hive internal column name ("+ fieldName
+ + ") and position encoding (" + fpos
+ + ") for the column name are at odds");
}
+ // If we reached here, then we were successful at finding an alternate internal
+ // column mapping, and we're about to proceed.
}
HCatFieldSchema hcatFieldSchema = s.getFields().get(fpos);
Object currField = extractCurrentField(p, null, hcatFieldSchema, false);
@@ -210,6 +218,27 @@ public class JsonSerDe implements SerDe
}
/**
+ * Utility method to extract (and forget) the next value token from the JsonParser,
+ * as a whole. The reason this function gets called is to yank out the next value altogether,
+ * because it corresponds to a field name that we do not recognize, and thus, do not have
+ * a schema/type for. Thus, this field is to be ignored.
+ * @throws IOException
+ * @throws JsonParseException
+ */
+ private void skipValue(JsonParser p) throws JsonParseException, IOException {
+ JsonToken valueToken = p.nextToken();
+
+ if ((valueToken == JsonToken.START_ARRAY) || (valueToken == JsonToken.START_OBJECT)){
+ // if the currently read token is a beginning of an array or object, move stream forward
+ // skipping any child tokens till we're at the corresponding END_ARRAY or END_OBJECT token
+ p.skipChildren();
+ }
+ // At the end of this function, the stream should be pointing to the last token that
+ // corresponds to the value being skipped. This way, the next call to nextToken
+ // will advance it to the next field name.
+ }
+
+ /**
* Utility method to extract current expected field from given JsonParser
*
* To get the field, we need either a type or a hcatFieldSchema(necessary for complex types)
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/DataTransferFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/DataTransferFactory.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/DataTransferFactory.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/DataTransferFactory.java Tue Jan 28 05:48:03 2014
@@ -21,8 +21,6 @@ package org.apache.hive.hcatalog.data.tr
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hive.hcatalog.data.transfer.impl.HCatInputFormatReader;
import org.apache.hive.hcatalog.data.transfer.impl.HCatOutputFormatWriter;
import org.apache.hive.hcatalog.data.transfer.state.DefaultStateProvider;
@@ -56,16 +54,16 @@ public class DataTransferFactory {
* This should only be called once from every slave node to obtain an instance
* of {@link HCatReader}.
*
- * @param split
- * input split obtained at master node
- * @param config
- * configuration obtained at master node
+ * @param context
+ * reader context obtained at the master node
+ * @param slaveNumber
+ * which slave this is, determines which part of the read is done
* @return {@link HCatReader}
*/
- public static HCatReader getHCatReader(final InputSplit split,
- final Configuration config) {
+ public static HCatReader getHCatReader(final ReaderContext context,
+ int slaveNumber) {
// In future, this may examine config to return appropriate HCatReader
- return getHCatReader(split, config, DefaultStateProvider.get());
+ return getHCatReader(context, slaveNumber, DefaultStateProvider.get());
}
/**
@@ -73,18 +71,19 @@ public class DataTransferFactory {
* of {@link HCatReader}. This should be called if an external system has some
* state to provide to HCatalog.
*
- * @param split
- * input split obtained at master node
- * @param config
- * configuration obtained at master node
+ * @param context
+ * reader context obtained at the master node
+ * @param slaveNumber
+ * which slave this is, determines which part of the read is done
* @param sp
* {@link StateProvider}
* @return {@link HCatReader}
*/
- public static HCatReader getHCatReader(final InputSplit split,
- final Configuration config, StateProvider sp) {
+ public static HCatReader getHCatReader(final ReaderContext context,
+ int slaveNumber,
+ StateProvider sp) {
// In future, this may examine config to return appropriate HCatReader
- return new HCatInputFormatReader(split, config, sp);
+ return new HCatInputFormatReader(context, slaveNumber, sp);
}
/**
@@ -131,6 +130,6 @@ public class DataTransferFactory {
public static HCatWriter getHCatWriter(final WriterContext cntxt,
final StateProvider sp) {
// In future, this may examine context to return appropriate HCatWriter
- return new HCatOutputFormatWriter(cntxt.getConf(), sp);
+ return new HCatOutputFormatWriter(cntxt, sp);
}
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/HCatReader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/HCatReader.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/HCatReader.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/HCatReader.java Tue Jan 28 05:48:03 2014
@@ -95,11 +95,4 @@ public abstract class HCatReader {
this.conf = conf;
}
- public Configuration getConf() {
- if (null == conf) {
- throw new IllegalStateException(
- "HCatReader is not constructed correctly.");
- }
- return conf;
- }
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/ReaderContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/ReaderContext.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/ReaderContext.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/ReaderContext.java Tue Jan 28 05:48:03 2014
@@ -20,70 +20,22 @@
package org.apache.hive.hcatalog.data.transfer;
import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hive.hcatalog.mapreduce.HCatSplit;
/**
- * This class will contain information of different {@link InputSplit} obtained
- * at master node and configuration. This class implements
- * {@link Externalizable} so it can be serialized using standard java
- * mechanisms.
+ * This read context is obtained by the master node and should be distributed
+ * to the slaves. The contents of the class are opaque to the client. This
+ * interface extends {@link java.io.Externalizable} so that implementing
+ * classes can be serialized using standard Java mechanisms.
*/
-public class ReaderContext implements Externalizable, Configurable {
+public interface ReaderContext extends Externalizable {
- private static final long serialVersionUID = -2656468331739574367L;
- private List<InputSplit> splits;
- private Configuration conf;
+ /**
+ * Determine the number of splits available in this {@link ReaderContext}.
+ * The client is not required to have this many slave nodes,
+ * as one slave can be used to read multiple splits.
+ * @return number of splits
+ */
+ public int numSplits();
- public ReaderContext() {
- this.splits = new ArrayList<InputSplit>();
- this.conf = new Configuration();
- }
-
- public void setInputSplits(final List<InputSplit> splits) {
- this.splits = splits;
- }
-
- public List<InputSplit> getSplits() {
- return splits;
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public void setConf(final Configuration config) {
- conf = config;
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- conf.write(out);
- out.writeInt(splits.size());
- for (InputSplit split : splits) {
- ((HCatSplit) split).write(out);
- }
- }
-
- @Override
- public void readExternal(ObjectInput in) throws IOException,
- ClassNotFoundException {
- conf.readFields(in);
- int numOfSplits = in.readInt();
- for (int i = 0; i < numOfSplits; i++) {
- HCatSplit split = new HCatSplit();
- split.readFields(in);
- splits.add(split);
- }
- }
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/WriterContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/WriterContext.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/WriterContext.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/WriterContext.java Tue Jan 28 05:48:03 2014
@@ -20,46 +20,14 @@
package org.apache.hive.hcatalog.data.transfer;
import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
/**
- * This contains information obtained at master node to help prepare slave nodes
- * for writer. This class implements {@link Externalizable} so it can be
- * serialized using standard java mechanisms. Master should serialize it and
+ * This contains information obtained at master node to be distributed to
+ * slaves nodes that will do the writing.
+ * This class implements {@link Externalizable} so it can be
+ * serialized using standard Java mechanisms. Master should serialize it and
* make it available to slaves to prepare for writes.
*/
-public class WriterContext implements Externalizable, Configurable {
-
- private static final long serialVersionUID = -5899374262971611840L;
- private Configuration conf;
-
- public WriterContext() {
- conf = new Configuration();
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public void setConf(final Configuration config) {
- this.conf = config;
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- conf.write(out);
- }
+public interface WriterContext extends Externalizable {
- @Override
- public void readExternal(ObjectInput in) throws IOException,
- ClassNotFoundException {
- conf.readFields(in);
- }
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatInputFormatReader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatInputFormatReader.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatInputFormatReader.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatInputFormatReader.java Tue Jan 28 05:48:03 2014
@@ -48,10 +48,10 @@ public class HCatInputFormatReader exten
private InputSplit split;
- public HCatInputFormatReader(InputSplit split, Configuration config,
- StateProvider sp) {
- super(config, sp);
- this.split = split;
+ public HCatInputFormatReader(ReaderContext context, int slaveNumber,
+ StateProvider sp) {
+ super(((ReaderContextImpl)context).getConf(), sp);
+ this.split = ((ReaderContextImpl)context).getSplits().get(slaveNumber);
}
public HCatInputFormatReader(ReadEntity info, Map<String, String> config) {
@@ -64,7 +64,7 @@ public class HCatInputFormatReader exten
Job job = new Job(conf);
HCatInputFormat hcif = HCatInputFormat.setInput(
job, re.getDbName(), re.getTableName(), re.getFilterString());
- ReaderContext cntxt = new ReaderContext();
+ ReaderContextImpl cntxt = new ReaderContextImpl();
cntxt.setInputSplits(hcif.getSplits(
ShimLoader.getHadoopShims().getHCatShim().createJobContext(job.getConfiguration(), null)));
cntxt.setConf(job.getConfiguration());
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java Tue Jan 28 05:48:03 2014
@@ -52,8 +52,8 @@ public class HCatOutputFormatWriter exte
super(we, config);
}
- public HCatOutputFormatWriter(Configuration config, StateProvider sp) {
- super(config, sp);
+ public HCatOutputFormatWriter(WriterContext cntxt, StateProvider sp) {
+ super(((WriterContextImpl)cntxt).getConf(), sp);
}
@Override
@@ -74,7 +74,7 @@ public class HCatOutputFormatWriter exte
} catch (InterruptedException e) {
throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
}
- WriterContext cntxt = new WriterContext();
+ WriterContextImpl cntxt = new WriterContextImpl();
cntxt.setConf(job.getConfiguration());
return cntxt;
}
@@ -124,10 +124,14 @@ public class HCatOutputFormatWriter exte
@Override
public void commit(WriterContext context) throws HCatException {
+ WriterContextImpl cntxtImpl = (WriterContextImpl)context;
try {
- new HCatOutputFormat().getOutputCommitter(ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptContext(
- context.getConf(), ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptID()))
- .commitJob(ShimLoader.getHadoopShims().getHCatShim().createJobContext(context.getConf(), null));
+ new HCatOutputFormat().getOutputCommitter(
+ ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptContext(
+ cntxtImpl.getConf(),
+ ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptID()))
+ .commitJob(ShimLoader.getHadoopShims().getHCatShim().createJobContext(
+ cntxtImpl.getConf(), null));
} catch (IOException e) {
throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
} catch (InterruptedException e) {
@@ -137,11 +141,14 @@ public class HCatOutputFormatWriter exte
@Override
public void abort(WriterContext context) throws HCatException {
+ WriterContextImpl cntxtImpl = (WriterContextImpl)context;
try {
- new HCatOutputFormat().getOutputCommitter(ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptContext(
- context.getConf(), ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptID()))
- .abortJob(ShimLoader.getHadoopShims().getHCatShim().createJobContext(
- context.getConf(), null), State.FAILED);
+ new HCatOutputFormat().getOutputCommitter(
+ ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptContext(
+ cntxtImpl.getConf(),
+ ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptID()))
+ .abortJob(ShimLoader.getHadoopShims().getHCatShim().createJobContext(
+ cntxtImpl.getConf(), null), State.FAILED);
} catch (IOException e) {
throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
} catch (InterruptedException e) {
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java Tue Jan 28 05:48:03 2014
@@ -74,6 +74,7 @@ import org.slf4j.LoggerFactory;
/**
* @deprecated Use/modify {@link org.apache.hive.hcatalog.mapreduce.TestHCatMultiOutputFormat} instead
*/
+@Deprecated
public class TestHCatMultiOutputFormat {
private static final Logger LOG = LoggerFactory.getLogger(TestHCatMultiOutputFormat.class);
@@ -377,13 +378,13 @@ public class TestHCatMultiOutputFormat {
org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table);
FetchWork work;
if (tbl.getPartCols().isEmpty()) {
- work = new FetchWork(new Path(tbl.getDataLocation()), Utilities.getTableDesc(tbl));
+ work = new FetchWork(tbl.getDataLocation(), Utilities.getTableDesc(tbl));
} else {
List<Partition> partitions = hive.getPartitions(tbl);
List<PartitionDesc> partDesc = new ArrayList<PartitionDesc>();
List<Path> partLocs = new ArrayList<Path>();
for (Partition part : partitions) {
- partLocs.add(part.getPartitionPath());
+ partLocs.add(part.getDataLocation());
partDesc.add(Utilities.getPartitionDesc(part));
}
work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl));
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java Tue Jan 28 05:48:03 2014
@@ -29,6 +29,7 @@ import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -211,4 +212,56 @@ public class TestJsonSerDe extends TestC
}
return sb.toString();
}
+
+ /**
+ * This test tests that our json deserialization is not too strict, as per HIVE-6166
+ *
+ * i.e, if our schema is "s:struct<a:int,b:string>,k:int", and we pass in
+ * data that looks like : {
+ * "x" : "abc" ,
+ * "t" : {
+ * "a" : "1",
+ * "b" : "2",
+ * "c" : [
+ * { "x" : 2 , "y" : 3 } ,
+ * { "x" : 3 , "y" : 2 }
+ * ]
+ * } ,
+ * "s" : {
+ * "a" : 2 ,
+ * "b" : "blah",
+ * "c": "woo"
+ * }
+ * }
+ *
+ * Then it should still work, and ignore the "x" and "t" field and "c" subfield of "s", and it
+ * should read k as null.
+ */
+ public void testLooseJsonReadability() throws Exception {
+ Configuration conf = new Configuration();
+ Properties props = new Properties();
+
+ props.put(serdeConstants.LIST_COLUMNS, "s,k");
+ props.put(serdeConstants.LIST_COLUMN_TYPES, "struct<a:int,b:string>,int");
+ JsonSerDe rjsd = new JsonSerDe();
+ rjsd.initialize(conf, props);
+
+ Text jsonText = new Text("{ \"x\" : \"abc\" , "
+ + " \"t\" : { \"a\":\"1\", \"b\":\"2\", \"c\":[ { \"x\":2 , \"y\":3 } , { \"x\":3 , \"y\":2 }] } ,"
+ +"\"s\" : { \"a\" : 2 , \"b\" : \"blah\", \"c\": \"woo\" } }");
+ List<Object> expected = new ArrayList<Object>();
+ List<Object> inner = new ArrayList<Object>();
+ inner.add(2);
+ inner.add("blah");
+ expected.add(inner);
+ expected.add(null);
+ HCatRecord expectedRecord = new DefaultHCatRecord(expected);
+
+ HCatRecord r = (HCatRecord) rjsd.deserialize(jsonText);
+ System.err.println("record : " + r.toString());
+
+ assertTrue(HCatDataCheckUtil.recordsEqual(r, expectedRecord));
+
+ }
+
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java Tue Jan 28 05:48:03 2014
@@ -95,8 +95,8 @@ public class TestReaderWriter extends HC
readCntxt = (ReaderContext) ois.readObject();
ois.close();
- for (InputSplit split : readCntxt.getSplits()) {
- runsInSlave(split, readCntxt.getConf());
+ for (int i = 0; i < readCntxt.numSplits(); i++) {
+ runsInSlave(readCntxt, i);
}
}
@@ -117,9 +117,9 @@ public class TestReaderWriter extends HC
return cntxt;
}
- private void runsInSlave(InputSplit split, Configuration config) throws HCatException {
+ private void runsInSlave(ReaderContext cntxt, int slaveNum) throws HCatException {
- HCatReader reader = DataTransferFactory.getHCatReader(split, config);
+ HCatReader reader = DataTransferFactory.getHCatReader(cntxt, slaveNum);
Iterator<HCatRecord> itr = reader.read();
int i = 1;
while (itr.hasNext()) {
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java Tue Jan 28 05:48:03 2014
@@ -381,13 +381,13 @@ public class TestHCatMultiOutputFormat {
List<PartitionDesc> partDesc = new ArrayList<PartitionDesc>();
List<Path> partLocs = new ArrayList<Path>();
for (Partition part : partitions) {
- partLocs.add(part.getPartitionPath());
+ partLocs.add(part.getDataLocation());
partDesc.add(Utilities.getPartitionDesc(part));
}
work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl));
work.setLimit(100);
} else {
- work = new FetchWork(new Path(tbl.getDataLocation()), Utilities.getTableDesc(tbl));
+ work = new FetchWork(tbl.getDataLocation(), Utilities.getTableDesc(tbl));
}
FetchTask task = new FetchTask();
task.setWork(work);
Propchange: hive/branches/tez/hcatalog/hcatalog-pig-adapter/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Propchange: hive/branches/tez/hcatalog/server-extensions/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Tue Jan 28 05:48:03 2014
@@ -650,13 +650,22 @@ sub compare
# decode $testResult->{'body'} to an array of hash
my $body = decode_json $testResult->{'body'};
- my @sorted_body;
+ my @filtered_body;
+ if (defined $testCmd->{'filter_job_names'}) {
+ foreach my $filter (@{$testCmd->{'filter_job_names'}}) {
+ my @filtered_body_tmp = grep { $_->{detail}{profile}{jobName} eq $filter } @$body;
+ @filtered_body = (@filtered_body, @filtered_body_tmp);
+ }
+ } else {
+ @filtered_body = @$body;
+ }
+ my @sorted_filtered_body;
if (ref @$body[0] eq 'HASH') {
- @sorted_body = sort { $a->{id} cmp $b->{id} } @$body;
+ @sorted_filtered_body = sort { $a->{id} cmp $b->{id} } @filtered_body;
} else {
- @sorted_body = sort { $a cmp $b } @$body;
+ @sorted_filtered_body = sort { $a cmp $b } @filtered_body;
}
- my $value = $path->value(\@sorted_body);
+ my $value = $path->value(\@sorted_filtered_body);
if ($value !~ /$regex_expected_value/s) {
print $log "$0::$subName INFO check failed:"
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf Tue Jan 28 05:48:03 2014
@@ -86,6 +86,7 @@ $cfg =
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&showall=true&fields=*',
'format_header' => 'Content-Type: application/json',
+ 'filter_job_names' => ['TempletonControllerJob', 'PigLatin:loadstore.pig'],
'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME:',
'$[-4:].detail.status.username' => ':UNAME:', '$[-5:].detail.status.username' => ':UNAME_OTHER:', '$[-6:].detail.status.username' => ':UNAME_OTHER:'},
'status_code' => 200,
@@ -97,6 +98,7 @@ $cfg =
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
'format_header' => 'Content-Type: application/json',
+ 'filter_job_names' => ['TempletonControllerJob', 'PigLatin:loadstore.pig'],
'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME_OTHER:',
'$[-4:].detail.status.username' => ':UNAME_OTHER:'},
'status_code' => 200,
@@ -118,6 +120,7 @@ $cfg =
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
'format_header' => 'Content-Type: application/json',
+ 'filter_job_names' => ['TempletonControllerJob', 'PigLatin:loadstore.pig'],
'json_path' => {'$[-1:].id' => 'job_.*',
'$[-1:].detail.status.jobId' => 'job_.*',
'$[-1:].detail.status.runState' => '\\d+',
Propchange: hive/branches/tez/hcatalog/storage-handlers/hbase/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Propchange: hive/branches/tez/hcatalog/webhcat/java-client/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Propchange: hive/branches/tez/hcatalog/webhcat/svr/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Propchange: hive/branches/tez/hwi/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java Tue Jan 28 05:48:03 2014
@@ -18,6 +18,10 @@
package org.apache.hive.beeline;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@@ -32,15 +36,12 @@ import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hive.beeline.BeeLine;
import org.apache.hive.service.server.HiveServer2;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.*;
-
/**
* TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine
*
@@ -282,6 +283,14 @@ public class TestBeeLineWithArgs {
testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL));
}
+ @Test
+ public void testGetVariableValue() throws Throwable {
+ final String TEST_NAME = "testGetVariableValue";
+ final String SCRIPT_TEXT = "set env:TERM;";
+ final String EXPECTED_PATTERN = "env:TERM";
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL));
+ }
+
/**
* Select null from table , check if setting null to empty string works.
* Original beeline/sqlline used to print nulls as empty strings
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Tue Jan 28 05:48:03 2014
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertNul
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.InputStream;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
@@ -187,7 +188,8 @@ public class TestJdbcDriver2 {
+ " c19 binary, "
+ " c20 date,"
+ " c21 varchar(20),"
- + " c22 char(15)"
+ + " c22 char(15),"
+ + " c23 binary"
+ ") comment'" + dataTypeTableComment
+"' partitioned by (dt STRING)");
@@ -443,7 +445,7 @@ public class TestJdbcDriver2 {
ResultSet res = stmt.executeQuery(
"explain select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " +
- "c1*2, sentences(null, null, null) as b from " + dataTypeTableName + " limit 1");
+ "c1*2, sentences(null, null, null) as b, c23 from " + dataTypeTableName + " limit 1");
ResultSetMetaData md = res.getMetaData();
// only one result column
@@ -821,6 +823,7 @@ public class TestJdbcDriver2 {
assertEquals(null, res.getDate(20));
assertEquals(null, res.getString(21));
assertEquals(null, res.getString(22));
+ assertEquals(null, res.getString(23));
// row 3
assertTrue(res.next());
@@ -849,6 +852,15 @@ public class TestJdbcDriver2 {
assertEquals("abc123", res.getString(21));
assertEquals("abc123 ", res.getString(22));
+ byte[] bytes = "X'01FF'".getBytes("UTF-8");
+ InputStream resultSetInputStream = res.getBinaryStream(23);
+ int len = bytes.length;
+ byte[] b = new byte[len];
+ resultSetInputStream.read(b, 0, len);
+ for ( int i = 0; i< len; i++) {
+ assertEquals(bytes[i], b[i]);
+ }
+
// test getBoolean rules on non-boolean columns
assertEquals(true, res.getBoolean(1));
assertEquals(true, res.getBoolean(4));
@@ -1329,14 +1341,14 @@ public class TestJdbcDriver2 {
ResultSet res = stmt.executeQuery(
"select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " +
- "c1*2, sentences(null, null, null) as b, c17, c18, c20, c21, c22 from " + dataTypeTableName +
+ "c1*2, sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from " + dataTypeTableName +
" limit 1");
ResultSetMetaData meta = res.getMetaData();
ResultSet colRS = con.getMetaData().getColumns(null, null,
dataTypeTableName.toLowerCase(), null);
- assertEquals(19, meta.getColumnCount());
+ assertEquals(20, meta.getColumnCount());
assertTrue(colRS.next());
@@ -1400,53 +1412,53 @@ public class TestJdbcDriver2 {
assertTrue(colRS.next());
assertEquals("a", meta.getColumnName(5));
- assertEquals(Types.VARCHAR, meta.getColumnType(5));
- assertEquals("string", meta.getColumnTypeName(5));
+ assertEquals(Types.ARRAY, meta.getColumnType(5));
+ assertEquals("array", meta.getColumnTypeName(5));
assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(5));
assertEquals(Integer.MAX_VALUE, meta.getPrecision(5));
assertEquals(0, meta.getScale(5));
assertEquals("c5", colRS.getString("COLUMN_NAME"));
- assertEquals(Types.VARCHAR, colRS.getInt("DATA_TYPE"));
+ assertEquals(Types.ARRAY, colRS.getInt("DATA_TYPE"));
assertEquals("array<int>", colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c6", meta.getColumnName(6));
- assertEquals(Types.VARCHAR, meta.getColumnType(6));
- assertEquals("string", meta.getColumnTypeName(6));
+ assertEquals(Types.JAVA_OBJECT, meta.getColumnType(6));
+ assertEquals("map", meta.getColumnTypeName(6));
assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(6));
assertEquals(Integer.MAX_VALUE, meta.getPrecision(6));
assertEquals(0, meta.getScale(6));
assertEquals("c6", colRS.getString("COLUMN_NAME"));
- assertEquals(Types.VARCHAR, colRS.getInt("DATA_TYPE"));
+ assertEquals(Types.JAVA_OBJECT, colRS.getInt("DATA_TYPE"));
assertEquals("map<int,string>", colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c7", meta.getColumnName(7));
- assertEquals(Types.VARCHAR, meta.getColumnType(7));
- assertEquals("string", meta.getColumnTypeName(7));
+ assertEquals(Types.JAVA_OBJECT, meta.getColumnType(7));
+ assertEquals("map", meta.getColumnTypeName(7));
assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(7));
assertEquals(Integer.MAX_VALUE, meta.getPrecision(7));
assertEquals(0, meta.getScale(7));
assertEquals("c7", colRS.getString("COLUMN_NAME"));
- assertEquals(Types.VARCHAR, colRS.getInt("DATA_TYPE"));
+ assertEquals(Types.JAVA_OBJECT, colRS.getInt("DATA_TYPE"));
assertEquals("map<string,string>", colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c8", meta.getColumnName(8));
- assertEquals(Types.VARCHAR, meta.getColumnType(8));
- assertEquals("string", meta.getColumnTypeName(8));
+ assertEquals(Types.STRUCT, meta.getColumnType(8));
+ assertEquals("struct", meta.getColumnTypeName(8));
assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(8));
assertEquals(Integer.MAX_VALUE, meta.getPrecision(8));
assertEquals(0, meta.getScale(8));
assertEquals("c8", colRS.getString("COLUMN_NAME"));
- assertEquals(Types.VARCHAR, colRS.getInt("DATA_TYPE"));
+ assertEquals(Types.STRUCT, colRS.getInt("DATA_TYPE"));
assertEquals("struct<r:string,s:int,t:double>", colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
@@ -1517,8 +1529,8 @@ public class TestJdbcDriver2 {
assertEquals(0, meta.getScale(13));
assertEquals("b", meta.getColumnName(14));
- assertEquals(Types.VARCHAR, meta.getColumnType(14));
- assertEquals("string", meta.getColumnTypeName(14));
+ assertEquals(Types.ARRAY, meta.getColumnType(14));
+ assertEquals("array", meta.getColumnTypeName(14));
assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(14));
assertEquals(Integer.MAX_VALUE, meta.getPrecision(14));
assertEquals(0, meta.getScale(14));
@@ -1560,6 +1572,13 @@ public class TestJdbcDriver2 {
assertEquals(15, meta.getPrecision(19));
assertEquals(0, meta.getScale(19));
+ assertEquals("c23", meta.getColumnName(20));
+ assertEquals(Types.BINARY, meta.getColumnType(20));
+ assertEquals("binary", meta.getColumnTypeName(20));
+ assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(20));
+ assertEquals(Integer.MAX_VALUE, meta.getPrecision(20));
+ assertEquals(0, meta.getScale(20));
+
for (int i = 1; i <= meta.getColumnCount(); i++) {
assertFalse(meta.isAutoIncrement(i));
assertFalse(meta.isCurrency(i));
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java Tue Jan 28 05:48:03 2014
@@ -163,5 +163,46 @@ import org.junit.Test;
stmt.execute(" drop table if exists table_in_non_default_schema");
expected = stmt.execute("DROP DATABASE "+ dbName);
stmt.close();
+
+ hs2Conn = DriverManager.getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ res = stmt.executeQuery("show tables");
+ testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
+ }
+
+ // test URI with no dbName
+ hs2Conn = DriverManager.getConnection(jdbcUri, System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = DriverManager.getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = DriverManager.getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+ }
+
+ /**
+ * verify that the current db is the one expected. first create table as <db>.tab and then
+ * describe that table to check if <db> is the current database
+ * @param expectedDbName
+ * @param hs2Conn
+ * @throws Exception
+ */
+ private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception {
+ String verifyTab = "miniHS2DbVerificationTable";
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)");
+ stmt.execute("DESCRIBE " + verifyTab);
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.close();
}
}
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java Tue Jan 28 05:48:03 2014
@@ -30,6 +30,7 @@ import org.apache.hive.service.cli.Opera
import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.SessionHandle;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -66,4 +67,14 @@ public class TestHiveServer2 {
RowSet rowSet = serviceClient.fetchResults(opHandle);
assertFalse(rowSet.numRows() == 0);
}
+
+ @Test
+ public void testGetVariableValue() throws Exception {
+ CLIServiceClient serviceClient = miniHS2.getServiceClient();
+ SessionHandle sessHandle = serviceClient.openSession("foo", "bar");
+ OperationHandle opHandle = serviceClient.executeStatement(sessHandle, "set system:os.name", confOverlay);
+ RowSet rowSet = serviceClient.fetchResults(opHandle);
+ Assert.assertEquals(1, rowSet.numRows());
+ }
+
}
Modified: hive/branches/tez/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/pom.xml?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/qtest/pom.xml (original)
+++ hive/branches/tez/itests/qtest/pom.xml Tue Jan 28 05:48:03 2014
@@ -36,7 +36,7 @@
<run_disabled>false</run_disabled>
<clustermode></clustermode>
<execute.beeline.tests>false</execute.beeline.tests>
- <minimr.query.files>stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q</minimr.query.files>
+ <minimr.query.files>stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q</minimr.query.files>
<minimr.query.negative.files>cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q</minimr.query.negative.files>
<minitez.query.files>tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q</minitez.query.files>
<minitez.query.files.shared>join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q</minitez.query.files.shared>
@@ -354,18 +354,6 @@
<version>${tez.version}</version>
<scope>test</scope>
</dependency>
- <dependency >
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-common</artifactId>
- <version>${tez.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-runtime-internals</artifactId>
- <version>${tez.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</profile>
</profiles>
Modified: hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Tue Jan 28 05:48:03 2014
@@ -51,6 +51,7 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -1709,4 +1710,31 @@ public class QTestUtil {
}
return result;
}
+
+ public void failed(int ecode, String fname, String debugHint) {
+ String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ Assert.fail("Client Execution failed with error code = " + ecode +
+ (command != null ? " running " + command : "") + (debugHint != null ? debugHint : ""));
+ }
+
+ public void failed(String fname, String debugHint) {
+ String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ Assert.fail("Client Execution was expected to fail, but succeeded with error code 0 " +
+ (command != null ? " running " + command : "") + (debugHint != null ? debugHint : ""));
+ }
+
+ public void failedDiff(int ecode, String fname, String debugHint) {
+ Assert.fail("Client Execution results failed with error code = " + ecode +
+ (debugHint != null ? debugHint : ""));
+ }
+
+ public void failed(Throwable e, String fname, String debugHint) {
+ String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.println("Failed query: " + fname);
+ System.err.flush();
+ Assert.fail("Unexpected exception" + (command != null ? " running " + command : "") +
+ (debugHint != null ? debugHint : ""));
+ }
}
Modified: hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyOutputTableLocationSchemeIsFileHook.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyOutputTableLocationSchemeIsFileHook.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyOutputTableLocationSchemeIsFileHook.java (original)
+++ hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyOutputTableLocationSchemeIsFileHook.java Tue Jan 28 05:48:03 2014
@@ -22,10 +22,11 @@ import org.junit.Assert;
public class VerifyOutputTableLocationSchemeIsFileHook implements ExecuteWithHookContext {
+ @Override
public void run(HookContext hookContext) {
for (WriteEntity output : hookContext.getOutputs()) {
if (output.getType() == WriteEntity.Type.TABLE) {
- String scheme = output.getTable().getDataLocation().getScheme();
+ String scheme = output.getTable().getDataLocation().toUri().getScheme();
Assert.assertTrue(output.getTable().getTableName() + " has a location which has a " +
"scheme other than file: " + scheme, scheme.equals("file"));
}
Modified: hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsNotSubdirectoryOfTableHook.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsNotSubdirectoryOfTableHook.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsNotSubdirectoryOfTableHook.java (original)
+++ hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsNotSubdirectoryOfTableHook.java Tue Jan 28 05:48:03 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
// not a subdirectory.
public class VerifyPartitionIsNotSubdirectoryOfTableHook implements ExecuteWithHookContext {
+ @Override
public void run(HookContext hookContext) {
for (WriteEntity output : hookContext.getOutputs()) {
if (output.getType() == WriteEntity.Type.PARTITION) {
@@ -44,6 +45,6 @@ public class VerifyPartitionIsNotSubdire
private void verify(Partition partition, Table table) {
Assert.assertFalse("The location of the partition: " + partition.getName() + " was a " +
"subdirectory of the location of the table: " + table.getTableName(),
- partition.getPartitionPath().toString().startsWith(table.getPath().toString()));
+ partition.getDataLocation().toString().startsWith(table.getPath().toString()));
}
}
Modified: hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsSubdirectoryOfTableHook.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsSubdirectoryOfTableHook.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsSubdirectoryOfTableHook.java (original)
+++ hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyPartitionIsSubdirectoryOfTableHook.java Tue Jan 28 05:48:03 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.metadat
// the location of the table. It is a very simple check to make sure it is a subdirectory.
public class VerifyPartitionIsSubdirectoryOfTableHook implements ExecuteWithHookContext {
+ @Override
public void run(HookContext hookContext) {
for (WriteEntity output : hookContext.getOutputs()) {
if (output.getType() == WriteEntity.Type.PARTITION) {
@@ -43,6 +44,6 @@ public class VerifyPartitionIsSubdirecto
private void verify(Partition partition, Table table) {
Assert.assertTrue("The location of the partition: " + partition.getName() + " was not a " +
"subdirectory of the location of the table: " + table.getTableName(),
- partition.getPartitionPath().toString().startsWith(table.getPath().toString()));
+ partition.getDataLocation().toString().startsWith(table.getPath().toString()));
}
}
Propchange: hive/branches/tez/jdbc/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java Tue Jan 28 05:48:03 2014
@@ -18,8 +18,10 @@
package org.apache.hive.jdbc;
+import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.Reader;
+import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.math.MathContext;
import java.net.URL;
@@ -134,11 +136,31 @@ public abstract class HiveBaseResultSet
}
public InputStream getBinaryStream(int columnIndex) throws SQLException {
- throw new SQLException("Method not supported");
+ Object obj = getObject(columnIndex);
+ if (obj == null) {
+ return null;
+ } else if (obj instanceof InputStream) {
+ return (InputStream)obj;
+ } else if (obj instanceof byte[]) {
+ byte[] byteArray = (byte[])obj;
+ InputStream is = new ByteArrayInputStream(byteArray);
+ return is;
+ } else if (obj instanceof String) {
+ String str = (String)obj;
+ InputStream is = null;
+ try {
+ is = new ByteArrayInputStream(str.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new SQLException("Illegal conversion to binary stream from column " +
+ columnIndex + " - Unsupported encoding exception");
+ }
+ return is;
+ }
+ throw new SQLException("Illegal conversion to binary stream from column " + columnIndex);
}
public InputStream getBinaryStream(String columnName) throws SQLException {
- throw new SQLException("Method not supported");
+ return getBinaryStream(findColumn(columnName));
}
public Blob getBlob(int i) throws SQLException {
@@ -410,6 +432,11 @@ public abstract class HiveBaseResultSet
return new BigDecimal((String)value);
case DATE_TYPE:
return Date.valueOf((String) value);
+ case ARRAY_TYPE:
+ case MAP_TYPE:
+ case STRUCT_TYPE:
+ // todo: returns json string. should recreate object from it?
+ return value;
default:
return value;
}
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Tue Jan 28 05:48:03 2014
@@ -297,8 +297,10 @@ public class HiveConnection implements j
for (Entry<String, String> hiveVar : hiveVarMap.entrySet()) {
stmt.execute("set hivevar:" + hiveVar.getKey() + "=" + hiveVar.getValue());
}
- if(dbName!=null)
- stmt.execute("use "+dbName);
+ // if the client is setting a non-default db, then switch the database
+ if (!Utils.DEFAULT_DATABASE.equalsIgnoreCase(dbName)) {
+ stmt.execute("use " + dbName);
+ }
stmt.close();
}
}
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java Tue Jan 28 05:48:03 2014
@@ -39,6 +39,7 @@ import java.sql.Time;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.HashMap;
+import java.util.Scanner;
import org.apache.hive.service.cli.thrift.TCLIService;
import org.apache.hive.service.cli.thrift.TSessionHandle;
@@ -263,8 +264,8 @@ public class HivePreparedStatement exten
*/
public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
+ String str = new Scanner(x, "UTF-8").useDelimiter("\\A").next();
+ this.parameters.put(parameterIndex, str);
}
/*
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java Tue Jan 28 05:48:03 2014
@@ -20,11 +20,8 @@ package org.apache.hive.jdbc;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
-import java.util.Arrays;
import java.util.List;
-import org.apache.hadoop.hive.serde.serdeConstants;
-
/**
* HiveResultSetMetaData.
*
@@ -47,7 +44,8 @@ public class HiveResultSetMetaData imple
}
public String getColumnClassName(int column) throws SQLException {
- throw new SQLException("Method not supported");
+ int columnType = getColumnType(column);
+ return JdbcColumn.columnClassName(columnType, columnAttributes.get(toZeroIndex(column)));
}
public int getColumnCount() throws SQLException {
@@ -57,91 +55,39 @@ public class HiveResultSetMetaData imple
public int getColumnDisplaySize(int column) throws SQLException {
int columnType = getColumnType(column);
- return JdbcColumn.columnDisplaySize(columnType, columnAttributes.get(column - 1));
+ return JdbcColumn.columnDisplaySize(columnType, columnAttributes.get(toZeroIndex(column)));
}
public String getColumnLabel(int column) throws SQLException {
- return columnNames.get(column - 1);
+ return columnNames.get(toZeroIndex(column));
}
public String getColumnName(int column) throws SQLException {
- return columnNames.get(column - 1);
+ return columnNames.get(toZeroIndex(column));
}
public int getColumnType(int column) throws SQLException {
- if (columnTypes == null) {
- throw new SQLException(
- "Could not determine column type name for ResultSet");
- }
-
- if (column < 1 || column > columnTypes.size()) {
- throw new SQLException("Invalid column value: " + column);
- }
-
// we need to convert the thrift type to the SQL type
- String type = columnTypes.get(column - 1);
+ String type = columnTypes.get(toZeroIndex(column));
// we need to convert the thrift type to the SQL type
- return Utils.hiveTypeToSqlType(type);
+ return JdbcColumn.hiveTypeToSqlType(type);
}
public String getColumnTypeName(int column) throws SQLException {
- validateColumnType(column);
-
- // we need to convert the Hive type to the SQL type name
- // TODO: this would be better handled in an enum
- String type = columnTypes.get(column - 1);
- if ("string".equalsIgnoreCase(type)) {
- return serdeConstants.STRING_TYPE_NAME;
- } else if ("varchar".equalsIgnoreCase(type)) {
- return serdeConstants.VARCHAR_TYPE_NAME;
- } else if ("char".equalsIgnoreCase(type)) {
- return serdeConstants.CHAR_TYPE_NAME;
- } else if ("float".equalsIgnoreCase(type)) {
- return serdeConstants.FLOAT_TYPE_NAME;
- } else if ("double".equalsIgnoreCase(type)) {
- return serdeConstants.DOUBLE_TYPE_NAME;
- } else if ("boolean".equalsIgnoreCase(type)) {
- return serdeConstants.BOOLEAN_TYPE_NAME;
- } else if ("tinyint".equalsIgnoreCase(type)) {
- return serdeConstants.TINYINT_TYPE_NAME;
- } else if ("smallint".equalsIgnoreCase(type)) {
- return serdeConstants.SMALLINT_TYPE_NAME;
- } else if ("int".equalsIgnoreCase(type)) {
- return serdeConstants.INT_TYPE_NAME;
- } else if ("bigint".equalsIgnoreCase(type)) {
- return serdeConstants.BIGINT_TYPE_NAME;
- } else if ("timestamp".equalsIgnoreCase(type)) {
- return serdeConstants.TIMESTAMP_TYPE_NAME;
- } else if ("date".equalsIgnoreCase(type)) {
- return serdeConstants.DATE_TYPE_NAME;
- } else if ("decimal".equalsIgnoreCase(type)) {
- return serdeConstants.DECIMAL_TYPE_NAME;
- } else if ("binary".equalsIgnoreCase(type)) {
- return serdeConstants.BINARY_TYPE_NAME;
- } else if ("void".equalsIgnoreCase(type)) {
- return serdeConstants.VOID_TYPE_NAME;
- } else if (type.startsWith("map<")) {
- return serdeConstants.STRING_TYPE_NAME;
- } else if (type.startsWith("array<")) {
- return serdeConstants.STRING_TYPE_NAME;
- } else if (type.startsWith("struct<")) {
- return serdeConstants.STRING_TYPE_NAME;
- }
-
- throw new SQLException("Unrecognized column type: " + type);
+ return JdbcColumn.getColumnTypeName(columnTypes.get(toZeroIndex(column)));
}
public int getPrecision(int column) throws SQLException {
int columnType = getColumnType(column);
- return JdbcColumn.columnPrecision(columnType, columnAttributes.get(column - 1));
+ return JdbcColumn.columnPrecision(columnType, columnAttributes.get(toZeroIndex(column)));
}
public int getScale(int column) throws SQLException {
int columnType = getColumnType(column);
- return JdbcColumn.columnScale(columnType, columnAttributes.get(column - 1));
+ return JdbcColumn.columnScale(columnType, columnAttributes.get(toZeroIndex(column)));
}
public String getSchemaName(int column) throws SQLException {
@@ -158,11 +104,9 @@ public class HiveResultSetMetaData imple
}
public boolean isCaseSensitive(int column) throws SQLException {
- validateColumnType(column);
-
// we need to convert the Hive type to the SQL type name
// TODO: this would be better handled in an enum
- String type = columnTypes.get(column - 1);
+ String type = columnTypes.get(toZeroIndex(column));
if("string".equalsIgnoreCase(type)) {
return true;
@@ -209,14 +153,14 @@ public class HiveResultSetMetaData imple
throw new SQLException("Method not supported");
}
- protected void validateColumnType(int column) throws SQLException {
+ protected int toZeroIndex(int column) throws SQLException {
if (columnTypes == null) {
throw new SQLException(
"Could not determine column type name for ResultSet");
}
-
if (column < 1 || column > columnTypes.size()) {
throw new SQLException("Invalid column value: " + column);
- }
+ }
+ return column - 1;
}
}
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java Tue Jan 28 05:48:03 2014
@@ -18,7 +18,12 @@
package org.apache.hive.jdbc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+
+import java.math.BigInteger;
+import java.sql.Date;
import java.sql.SQLException;
+import java.sql.Timestamp;
import java.sql.Types;
@@ -59,8 +64,125 @@ public class JdbcColumn {
return type;
}
- public Integer getSqlType() throws SQLException {
- return Utils.hiveTypeToSqlType(type);
+ static String columnClassName(int columnType, JdbcColumnAttributes columnAttributes)
+ throws SQLException {
+ // according to hiveTypeToSqlType possible options are:
+ switch(columnType) {
+ case Types.BOOLEAN:
+ return Boolean.class.getName();
+ case Types.CHAR:
+ case Types.VARCHAR:
+ return String.class.getName();
+ case Types.TINYINT:
+ return Byte.class.getName();
+ case Types.SMALLINT:
+ return Short.class.getName();
+ case Types.INTEGER:
+ return Integer.class.getName();
+ case Types.BIGINT:
+ return Long.class.getName();
+ case Types.DATE:
+ return Date.class.getName();
+ case Types.FLOAT:
+ return Float.class.getName();
+ case Types.DOUBLE:
+ return Double.class.getName();
+ case Types.TIMESTAMP:
+ return Timestamp.class.getName();
+ case Types.DECIMAL:
+ return BigInteger.class.getName();
+ case Types.BINARY:
+ return byte[].class.getName();
+ case Types.JAVA_OBJECT:
+ case Types.ARRAY:
+ case Types.STRUCT:
+ return String.class.getName();
+ default:
+ throw new SQLException("Invalid column type: " + columnType);
+ }
+ }
+
+ public static int hiveTypeToSqlType(String type) throws SQLException {
+ if ("string".equalsIgnoreCase(type)) {
+ return Types.VARCHAR;
+ } else if ("varchar".equalsIgnoreCase(type)) {
+ return Types.VARCHAR;
+ } else if ("char".equalsIgnoreCase(type)) {
+ return Types.CHAR;
+ } else if ("float".equalsIgnoreCase(type)) {
+ return Types.FLOAT;
+ } else if ("double".equalsIgnoreCase(type)) {
+ return Types.DOUBLE;
+ } else if ("boolean".equalsIgnoreCase(type)) {
+ return Types.BOOLEAN;
+ } else if ("tinyint".equalsIgnoreCase(type)) {
+ return Types.TINYINT;
+ } else if ("smallint".equalsIgnoreCase(type)) {
+ return Types.SMALLINT;
+ } else if ("int".equalsIgnoreCase(type)) {
+ return Types.INTEGER;
+ } else if ("bigint".equalsIgnoreCase(type)) {
+ return Types.BIGINT;
+ } else if ("date".equalsIgnoreCase(type)) {
+ return Types.DATE;
+ } else if ("timestamp".equalsIgnoreCase(type)) {
+ return Types.TIMESTAMP;
+ } else if ("decimal".equalsIgnoreCase(type)) {
+ return Types.DECIMAL;
+ } else if ("binary".equalsIgnoreCase(type)) {
+ return Types.BINARY;
+ } else if ("map".equalsIgnoreCase(type)) {
+ return Types.JAVA_OBJECT;
+ } else if ("array".equalsIgnoreCase(type)) {
+ return Types.ARRAY;
+ } else if ("struct".equalsIgnoreCase(type)) {
+ return Types.STRUCT;
+ }
+ throw new SQLException("Unrecognized column type: " + type);
+ }
+
+ static String getColumnTypeName(String type) throws SQLException {
+ // we need to convert the Hive type to the SQL type name
+ // TODO: this would be better handled in an enum
+ if ("string".equalsIgnoreCase(type)) {
+ return serdeConstants.STRING_TYPE_NAME;
+ } else if ("varchar".equalsIgnoreCase(type)) {
+ return serdeConstants.VARCHAR_TYPE_NAME;
+ } else if ("char".equalsIgnoreCase(type)) {
+ return serdeConstants.CHAR_TYPE_NAME;
+ } else if ("float".equalsIgnoreCase(type)) {
+ return serdeConstants.FLOAT_TYPE_NAME;
+ } else if ("double".equalsIgnoreCase(type)) {
+ return serdeConstants.DOUBLE_TYPE_NAME;
+ } else if ("boolean".equalsIgnoreCase(type)) {
+ return serdeConstants.BOOLEAN_TYPE_NAME;
+ } else if ("tinyint".equalsIgnoreCase(type)) {
+ return serdeConstants.TINYINT_TYPE_NAME;
+ } else if ("smallint".equalsIgnoreCase(type)) {
+ return serdeConstants.SMALLINT_TYPE_NAME;
+ } else if ("int".equalsIgnoreCase(type)) {
+ return serdeConstants.INT_TYPE_NAME;
+ } else if ("bigint".equalsIgnoreCase(type)) {
+ return serdeConstants.BIGINT_TYPE_NAME;
+ } else if ("timestamp".equalsIgnoreCase(type)) {
+ return serdeConstants.TIMESTAMP_TYPE_NAME;
+ } else if ("date".equalsIgnoreCase(type)) {
+ return serdeConstants.DATE_TYPE_NAME;
+ } else if ("decimal".equalsIgnoreCase(type)) {
+ return serdeConstants.DECIMAL_TYPE_NAME;
+ } else if ("binary".equalsIgnoreCase(type)) {
+ return serdeConstants.BINARY_TYPE_NAME;
+ } else if ("void".equalsIgnoreCase(type)) {
+ return serdeConstants.VOID_TYPE_NAME;
+ } else if (type.equalsIgnoreCase("map")) {
+ return serdeConstants.MAP_TYPE_NAME;
+ } else if (type.equalsIgnoreCase("array")) {
+ return serdeConstants.LIST_TYPE_NAME;
+ } else if (type.equalsIgnoreCase("struct")) {
+ return serdeConstants.STRUCT_TYPE_NAME;
+ }
+
+ throw new SQLException("Unrecognized column type: " + type);
}
static int columnDisplaySize(int columnType, JdbcColumnAttributes columnAttributes)
@@ -72,6 +194,8 @@ public class JdbcColumn {
case Types.CHAR:
case Types.VARCHAR:
return columnPrecision(columnType, columnAttributes);
+ case Types.BINARY:
+ return Integer.MAX_VALUE; // hive has no max limit for binary
case Types.TINYINT:
case Types.SMALLINT:
case Types.INTEGER:
@@ -90,6 +214,10 @@ public class JdbcColumn {
return 25; // e.g. -(17#).e-####
case Types.DECIMAL:
return columnPrecision(columnType, columnAttributes) + 2; // '-' sign and '.'
+ case Types.JAVA_OBJECT:
+ case Types.ARRAY:
+ case Types.STRUCT:
+ return Integer.MAX_VALUE;
default:
throw new SQLException("Invalid column type: " + columnType);
}
@@ -107,6 +235,8 @@ public class JdbcColumn {
return columnAttributes.precision;
}
return Integer.MAX_VALUE; // hive has no max limit for strings
+ case Types.BINARY:
+ return Integer.MAX_VALUE; // hive has no max limit for binary
case Types.TINYINT:
return 3;
case Types.SMALLINT:
@@ -125,6 +255,10 @@ public class JdbcColumn {
return 29;
case Types.DECIMAL:
return columnAttributes.precision;
+ case Types.JAVA_OBJECT:
+ case Types.ARRAY:
+ case Types.STRUCT:
+ return Integer.MAX_VALUE;
default:
throw new SQLException("Invalid column type: " + columnType);
}
@@ -142,6 +276,7 @@ public class JdbcColumn {
case Types.INTEGER:
case Types.BIGINT:
case Types.DATE:
+ case Types.BINARY:
return 0;
case Types.FLOAT:
return 7;
@@ -151,6 +286,10 @@ public class JdbcColumn {
return 9;
case Types.DECIMAL:
return columnAttributes.scale;
+ case Types.JAVA_OBJECT:
+ case Types.ARRAY:
+ case Types.STRUCT:
+ return 0;
default:
throw new SQLException("Invalid column type: " + columnType);
}
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/Utils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/Utils.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/Utils.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/Utils.java Tue Jan 28 05:48:03 2014
@@ -104,52 +104,6 @@ public class Utils {
}
}
-
- /**
- * Convert hive types to sql types.
- * @param type
- * @return Integer java.sql.Types values
- * @throws SQLException
- */
- public static int hiveTypeToSqlType(String type) throws SQLException {
- if ("string".equalsIgnoreCase(type)) {
- return Types.VARCHAR;
- } else if ("varchar".equalsIgnoreCase(type)) {
- return Types.VARCHAR;
- } else if ("char".equalsIgnoreCase(type)) {
- return Types.CHAR;
- } else if ("float".equalsIgnoreCase(type)) {
- return Types.FLOAT;
- } else if ("double".equalsIgnoreCase(type)) {
- return Types.DOUBLE;
- } else if ("boolean".equalsIgnoreCase(type)) {
- return Types.BOOLEAN;
- } else if ("tinyint".equalsIgnoreCase(type)) {
- return Types.TINYINT;
- } else if ("smallint".equalsIgnoreCase(type)) {
- return Types.SMALLINT;
- } else if ("int".equalsIgnoreCase(type)) {
- return Types.INTEGER;
- } else if ("bigint".equalsIgnoreCase(type)) {
- return Types.BIGINT;
- } else if ("date".equalsIgnoreCase(type)) {
- return Types.DATE;
- } else if ("timestamp".equalsIgnoreCase(type)) {
- return Types.TIMESTAMP;
- } else if ("decimal".equalsIgnoreCase(type)) {
- return Types.DECIMAL;
- } else if ("binary".equalsIgnoreCase(type)) {
- return Types.BINARY;
- } else if (type.startsWith("map<")) {
- return Types.VARCHAR;
- } else if (type.startsWith("array<")) {
- return Types.VARCHAR;
- } else if (type.startsWith("struct<")) {
- return Types.VARCHAR;
- }
- throw new SQLException("Unrecognized column type: " + type);
- }
-
// Verify success or success_with_info status, else throw SQLException
public static void verifySuccessWithInfo(TStatus status) throws SQLException {
verifySuccess(status, true);
@@ -229,17 +183,16 @@ public class Utils {
// dbname and session settings
String sessVars = jdbcURI.getPath();
- if ((sessVars == null) || sessVars.isEmpty()) {
- connParams.setDbName(DEFAULT_DATABASE);
- } else {
+ if ((sessVars != null) && !sessVars.isEmpty()) {
+ String dbName = "";
// removing leading '/' returned by getPath()
sessVars = sessVars.substring(1);
if (!sessVars.contains(";")) {
// only dbname is provided
- connParams.setDbName(sessVars);
+ dbName = sessVars;
} else {
// we have dbname followed by session parameters
- connParams.setDbName(sessVars.substring(0, sessVars.indexOf(';')));
+ dbName = sessVars.substring(0, sessVars.indexOf(';'));
sessVars = sessVars.substring(sessVars.indexOf(';')+1);
if (sessVars != null) {
Matcher sessMatcher = pattern.matcher(sessVars);
@@ -248,6 +201,9 @@ public class Utils {
}
}
}
+ if (!dbName.isEmpty()) {
+ connParams.setDbName(dbName);
+ }
}
// parse hive conf settings
Propchange: hive/branches/tez/metastore/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 28 05:48:03 2014
@@ -0,0 +1 @@
+target
Modified: hive/branches/tez/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/if/hive_metastore.thrift?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/tez/metastore/if/hive_metastore.thrift Tue Jan 28 05:48:03 2014
@@ -273,7 +273,7 @@ struct EnvironmentContext {
// Return type for get_partitions_by_expr
struct PartitionsByExprResult {
- 1: required set<Partition> partitions,
+ 1: required list<Partition> partitions,
// Whether the results has any (currently, all) partitions which may or may not match
2: required bool hasUnknownPartitions
}
@@ -286,6 +286,20 @@ struct PartitionsByExprRequest {
5: optional i16 maxParts=-1
}
+// Return type for add_partitions_req
+struct AddPartitionsResult {
+ 1: optional list<Partition> partitions,
+}
+
+// Request type for add_partitions_req
+struct AddPartitionsRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required list<Partition> parts,
+ 4: required bool ifNotExists,
+ 5: optional bool needResult=true
+}
+
exception MetaException {
1: string message
}
@@ -445,6 +459,8 @@ service ThriftHiveMetastore extends fb30
throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
Partition append_partition_with_environment_context(1:string db_name, 2:string tbl_name,
3:list<string> part_vals, 4:EnvironmentContext environment_context)
throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
Modified: hive/branches/tez/metastore/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/pom.xml?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/metastore/pom.xml (original)
+++ hive/branches/tez/metastore/pom.xml Tue Jan 28 05:48:03 2014
@@ -222,6 +222,7 @@
<api>JDO</api>
<verbose>true</verbose>
<metadataIncludes>**/*.jdo</metadataIncludes>
+ <fork>false</fork>
</configuration>
<executions>
<execution>