You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2010/01/21 11:38:15 UTC
svn commit: r901644 [5/37] - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/java/org/apache/hadoop/hive/ql/history/ ql/src/java...
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java Thu Jan 21 10:37:58 2010
@@ -21,36 +21,38 @@
import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
public class ExprNodeEvaluatorFactory {
-
- public ExprNodeEvaluatorFactory() {}
+
+ public ExprNodeEvaluatorFactory() {
+ }
public static ExprNodeEvaluator get(exprNodeDesc desc) {
// Constant node
if (desc instanceof exprNodeConstantDesc) {
- return new ExprNodeConstantEvaluator((exprNodeConstantDesc)desc);
+ return new ExprNodeConstantEvaluator((exprNodeConstantDesc) desc);
}
// Column-reference node, e.g. a column in the input row
if (desc instanceof exprNodeColumnDesc) {
- return new ExprNodeColumnEvaluator((exprNodeColumnDesc)desc);
+ return new ExprNodeColumnEvaluator((exprNodeColumnDesc) desc);
}
// Generic Function node, e.g. CASE, an operator or a UDF node
if (desc instanceof exprNodeGenericFuncDesc) {
- return new ExprNodeGenericFuncEvaluator((exprNodeGenericFuncDesc)desc);
+ return new ExprNodeGenericFuncEvaluator((exprNodeGenericFuncDesc) desc);
}
// Field node, e.g. get a.myfield1 from a
if (desc instanceof exprNodeFieldDesc) {
- return new ExprNodeFieldEvaluator((exprNodeFieldDesc)desc);
+ return new ExprNodeFieldEvaluator((exprNodeFieldDesc) desc);
}
- // Null node, a constant node with value NULL and no type information
+ // Null node, a constant node with value NULL and no type information
if (desc instanceof exprNodeNullDesc) {
- return new ExprNodeNullEvaluator((exprNodeNullDesc)desc);
+ return new ExprNodeNullEvaluator((exprNodeNullDesc) desc);
}
- throw new RuntimeException("Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc);
+ throw new RuntimeException(
+ "Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc);
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java Thu Jan 21 10:37:58 2010
@@ -23,8 +23,6 @@
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
-
-import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -32,9 +30,9 @@
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
/**
- * This Evaluator can evaluate s.f for s as both struct and list of struct.
- * If s is struct, then s.f is the field.
- * If s is list of struct, then s.f is the list of struct field.
+ * This Evaluator can evaluate s.f for s as both struct and list of struct. If s
+ * is struct, then s.f is the field. If s is list of struct, then s.f is the
+ * list of struct field.
*/
public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
@@ -45,7 +43,7 @@
transient StructField field;
transient ObjectInspector structFieldObjectInspector;
transient ObjectInspector resultObjectInspector;
-
+
public ExprNodeFieldEvaluator(exprNodeFieldDesc desc) {
this.desc = desc;
leftEvaluator = ExprNodeEvaluatorFactory.get(desc.getDesc());
@@ -54,7 +52,7 @@
@Override
public ObjectInspector initialize(ObjectInspector rowInspector)
throws HiveException {
-
+
leftInspector = leftEvaluator.initialize(rowInspector);
if (desc.getIsList()) {
structObjectInspector = (StructObjectInspector) ((ListObjectInspector) leftInspector)
@@ -73,22 +71,24 @@
}
return resultObjectInspector;
}
-
+
List<Object> cachedList = new ArrayList<Object>();
+
@Override
public Object evaluate(Object row) throws HiveException {
-
+
// Get the result in leftInspectableObject
Object left = leftEvaluator.evaluate(row);
if (desc.getIsList()) {
- List<?> list = ((ListObjectInspector)leftInspector).getList(left);
+ List<?> list = ((ListObjectInspector) leftInspector).getList(left);
if (list == null) {
return null;
} else {
cachedList.clear();
- for(int i=0; i<list.size(); i++) {
- cachedList.add(structObjectInspector.getStructFieldData(list.get(i), field));
+ for (int i = 0; i < list.size(); i++) {
+ cachedList.add(structObjectInspector.getStructFieldData(list.get(i),
+ field));
}
return cachedList;
}
@@ -97,5 +97,4 @@
}
}
-
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Thu Jan 21 10:37:58 2010
@@ -24,42 +24,43 @@
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.util.ReflectionUtils;
public class ExprNodeGenericFuncEvaluator extends ExprNodeEvaluator {
- private static final Log LOG = LogFactory.getLog(ExprNodeGenericFuncEvaluator.class.getName());
-
+ private static final Log LOG = LogFactory
+ .getLog(ExprNodeGenericFuncEvaluator.class.getName());
+
protected exprNodeGenericFuncDesc expr;
-
+
transient GenericUDF genericUDF;
transient Object rowObject;
transient ExprNodeEvaluator[] children;
transient DeferredExprObject[] deferredChildren;
-
+
/**
* Class to allow deferred evaluation for GenericUDF.
*/
class DeferredExprObject implements GenericUDF.DeferredObject {
-
- ExprNodeEvaluator eval;
+
+ ExprNodeEvaluator eval;
+
DeferredExprObject(ExprNodeEvaluator eval) {
this.eval = eval;
}
-
+
public Object get() throws HiveException {
return eval.evaluate(rowObject);
}
};
-
+
public ExprNodeGenericFuncEvaluator(exprNodeGenericFuncDesc expr) {
this.expr = expr;
children = new ExprNodeEvaluator[expr.getChildExprs().size()];
- for(int i=0; i<children.length; i++) {
+ for (int i = 0; i < children.length; i++) {
children[i] = ExprNodeEvaluatorFactory.get(expr.getChildExprs().get(i));
}
deferredChildren = new DeferredExprObject[expr.getChildExprs().size()];
- for(int i=0; i<deferredChildren.length; i++) {
+ for (int i = 0; i < deferredChildren.length; i++) {
deferredChildren[i] = new DeferredExprObject(children[i]);
}
}
@@ -69,16 +70,16 @@
throws HiveException {
// Initialize all children first
ObjectInspector[] childrenOIs = new ObjectInspector[children.length];
- for (int i=0; i<children.length; i++) {
+ for (int i = 0; i < children.length; i++) {
childrenOIs[i] = children[i].initialize(rowInspector);
}
genericUDF = expr.getGenericUDF();
return genericUDF.initialize(childrenOIs);
}
-
+
@Override
public Object evaluate(Object row) throws HiveException {
- this.rowObject = row;
+ rowObject = row;
return genericUDF.evaluate(deferredChildren);
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java Thu Jan 21 10:37:58 2010
@@ -35,7 +35,7 @@
@Override
public ObjectInspector initialize(ObjectInspector rowInspector)
- throws HiveException {
+ throws HiveException {
return PrimitiveObjectInspectorFactory.writableVoidObjectInspector;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java Thu Jan 21 10:37:58 2010
@@ -26,23 +26,26 @@
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
/**
- * Extract operator implementation
- * Extracts a subobject and passes that on.
+ * Extract operator implementation Extracts a subobject and passes that on.
**/
-public class ExtractOperator extends Operator<extractDesc> implements Serializable {
+public class ExtractOperator extends Operator<extractDesc> implements
+ Serializable {
private static final long serialVersionUID = 1L;
transient protected ExprNodeEvaluator eval;
+ @Override
protected void initializeOp(Configuration hconf) throws HiveException {
eval = ExprNodeEvaluatorFactory.get(conf.getCol());
outputObjInspector = eval.initialize(inputObjInspectors[0]);
initializeChildren(hconf);
}
+ @Override
public void processOp(Object row, int tag) throws HiveException {
forward(eval.evaluate(row), outputObjInspector);
}
+ @Override
public int getType() {
return OperatorType.EXTRACT;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Thu Jan 21 10:37:58 2010
@@ -30,29 +30,28 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.fetchWork;
import org.apache.hadoop.hive.ql.plan.partitionDesc;
import org.apache.hadoop.hive.ql.plan.tableDesc;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
/**
* FetchTask implementation
@@ -68,7 +67,7 @@
this.work = work;
this.job = job;
-
+
currRecReader = null;
currPath = null;
currTbl = null;
@@ -79,12 +78,12 @@
rowWithPart = new Object[2];
}
- private fetchWork work;
+ private final fetchWork work;
private int splitNum;
private RecordReader<WritableComparable, Writable> currRecReader;
private InputSplit[] inputSplits;
private InputFormat inputFormat;
- private JobConf job;
+ private final JobConf job;
private WritableComparable key;
private Writable value;
private Deserializer serde;
@@ -95,7 +94,7 @@
private tableDesc currTbl;
private boolean tblDataDone;
private StructObjectInspector rowObjectInspector;
- private Object[] rowWithPart;
+ private final Object[] rowWithPart;
/**
* A cache of InputFormat instances.
@@ -111,7 +110,7 @@
inputFormats.put(inputFormatClass, newInstance);
} catch (Exception e) {
throw new IOException("Cannot create an instance of InputFormat class "
- + inputFormatClass.getName() + " as specified in mapredWork!");
+ + inputFormatClass.getName() + " as specified in mapredWork!");
}
}
return inputFormats.get(inputFormatClass);
@@ -133,7 +132,8 @@
for (String key : partKeys) {
partNames.add(key);
partValues.add(partSpec.get(key));
- partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ partObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
}
StructObjectInspector partObjectInspector = ObjectInspectorFactory
.getStandardStructObjectInspector(partNames, partObjectInspectors);
@@ -164,8 +164,9 @@
}
}
- if (!tblDataDone)
+ if (!tblDataDone) {
currPath = null;
+ }
return;
} else {
currTbl = null;
@@ -173,7 +174,8 @@
}
return;
} else {
- iterPath = fetchWork.convertStringToPathArray(work.getPartDir()).iterator();
+ iterPath = fetchWork.convertStringToPathArray(work.getPartDir())
+ .iterator();
iterPartDesc = work.getPartDesc().iterator();
}
}
@@ -199,28 +201,33 @@
throws Exception {
if (currPath == null) {
getNextPath();
- if (currPath == null)
+ if (currPath == null) {
return null;
+ }
- // not using FileInputFormat.setInputPaths() here because it forces a connection
- // to the default file system - which may or may not be online during pure metadata
+ // not using FileInputFormat.setInputPaths() here because it forces a
+ // connection
+ // to the default file system - which may or may not be online during pure
+ // metadata
// operations
- job.set("mapred.input.dir",
- org.apache.hadoop.util.StringUtils.escapeString(currPath.toString()));
-
+ job.set("mapred.input.dir", org.apache.hadoop.util.StringUtils
+ .escapeString(currPath.toString()));
+
tableDesc tmp = currTbl;
- if (tmp == null)
+ if (tmp == null) {
tmp = currPart.getTableDesc();
+ }
inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
inputSplits = inputFormat.getSplits(job, 1);
splitNum = 0;
serde = tmp.getDeserializerClass().newInstance();
serde.initialize(job, tmp.getProperties());
LOG.debug("Creating fetchTask with deserializer typeinfo: "
- + serde.getObjectInspector().getTypeName());
+ + serde.getObjectInspector().getTypeName());
LOG.debug("deserializer properties: " + tmp.getProperties());
- if (!tblDataDone)
+ if (!tblDataDone) {
setPrtnDesc();
+ }
}
if (splitNum >= inputSplits.length) {
@@ -241,14 +248,15 @@
/**
* Get the next row. The fetch context is modified appropriately.
- *
+ *
**/
public InspectableObject getNextRow() throws IOException {
try {
if (currRecReader == null) {
currRecReader = getRecordReader();
- if (currRecReader == null)
+ if (currRecReader == null) {
return null;
+ }
}
boolean ret = currRecReader.next(key, value);
@@ -264,10 +272,11 @@
currRecReader.close();
currRecReader = null;
currRecReader = getRecordReader();
- if (currRecReader == null)
+ if (currRecReader == null) {
return null;
- else
+ } else {
return getNextRow();
+ }
}
} catch (Exception e) {
throw new IOException(e);
@@ -292,14 +301,12 @@
public ObjectInspector getOutputObjectInspector() throws HiveException {
try {
- ObjectInspector outInspector;
if (work.getTblDir() != null) {
tableDesc tbl = work.getTblDesc();
Deserializer serde = tbl.getDeserializerClass().newInstance();
serde.initialize(job, tbl.getProperties());
return serde.getObjectInspector();
- }
- else {
+ } else {
List<partitionDesc> listParts = work.getPartDesc();
currPart = listParts.get(0);
serde = currPart.getTableDesc().getDeserializerClass().newInstance();
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Thu Jan 21 10:37:58 2010
@@ -44,41 +44,44 @@
private int maxRows = 100;
private FetchOperator ftOp;
- private LazySimpleSerDe mSerde;
- private int totalRows;
-
- public FetchTask() {
- super();
- }
-
- @Override
- public void initialize (HiveConf conf, QueryPlan queryPlan, DriverContext ctx) {
+ private LazySimpleSerDe mSerde;
+ private int totalRows;
+
+ public FetchTask() {
+ super();
+ }
+
+ @Override
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx) {
super.initialize(conf, queryPlan, ctx);
-
- try {
- // Create a file system handle
- JobConf job = new JobConf(conf, ExecDriver.class);
-
- mSerde = new LazySimpleSerDe();
- Properties mSerdeProp = new Properties();
- mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
- mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, ((fetchWork)work).getSerializationNullFormat());
- mSerde.initialize(job, mSerdeProp);
- mSerde.setUseJSONSerialize(true);
-
- ftOp = new FetchOperator(work, job);
+
+ try {
+ // Create a file system handle
+ JobConf job = new JobConf(conf, ExecDriver.class);
+
+ mSerde = new LazySimpleSerDe();
+ Properties mSerdeProp = new Properties();
+ mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+ mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, (work)
+ .getSerializationNullFormat());
+ mSerde.initialize(job, mSerdeProp);
+ mSerde.setUseJSONSerialize(true);
+
+ ftOp = new FetchOperator(work, job);
} catch (Exception e) {
// Bail out ungracefully - we should never hit
// this here - but would have hit it in SemanticAnalyzer
LOG.error(StringUtils.stringifyException(e));
- throw new RuntimeException (e);
+ throw new RuntimeException(e);
}
}
-
+
+ @Override
public int execute() {
assert false;
return 0;
}
+
/**
* Return the tableDesc of the fetchWork
*/
@@ -99,41 +102,43 @@
public void setMaxRows(int maxRows) {
this.maxRows = maxRows;
}
-
+
+ @Override
public boolean fetch(Vector<String> res) throws IOException {
try {
int numRows = 0;
int rowsRet = maxRows;
- if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet))
+ if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet)) {
rowsRet = work.getLimit() - totalRows;
+ }
if (rowsRet <= 0) {
ftOp.clearFetchContext();
return false;
}
- while (numRows < rowsRet) {
- InspectableObject io = ftOp.getNextRow();
- if (io == null) {
- if (numRows == 0)
+ while (numRows < rowsRet) {
+ InspectableObject io = ftOp.getNextRow();
+ if (io == null) {
+ if (numRows == 0) {
return false;
+ }
totalRows += numRows;
return true;
- }
-
- res.add(((Text)mSerde.serialize(io.o, io.oi)).toString());
- numRows++;
- }
+ }
+
+ res.add(((Text) mSerde.serialize(io.o, io.oi)).toString());
+ numRows++;
+ }
totalRows += numRows;
return true;
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw e;
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new IOException(e);
}
}
-
+
+ @Override
public int getType() {
return StageType.FETCH;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Thu Jan 21 10:37:58 2010
@@ -41,10 +41,12 @@
/**
* File Sink operator implementation
**/
-public class FileSinkOperator extends TerminalOperator <fileSinkDesc> implements Serializable {
+public class FileSinkOperator extends TerminalOperator<fileSinkDesc> implements
+ Serializable {
public static interface RecordWriter {
public void write(Writable w) throws IOException;
+
public void close(boolean abort) throws IOException;
}
@@ -56,53 +58,57 @@
transient protected Serializer serializer;
transient protected BytesWritable commonKey = new BytesWritable();
transient protected TableIdEnum tabIdEnum = null;
- transient private LongWritable row_count;
+ transient private LongWritable row_count;
+
public static enum TableIdEnum {
TABLE_ID_1_ROWCOUNT, TABLE_ID_2_ROWCOUNT, TABLE_ID_3_ROWCOUNT, TABLE_ID_4_ROWCOUNT, TABLE_ID_5_ROWCOUNT, TABLE_ID_6_ROWCOUNT, TABLE_ID_7_ROWCOUNT, TABLE_ID_8_ROWCOUNT, TABLE_ID_9_ROWCOUNT, TABLE_ID_10_ROWCOUNT, TABLE_ID_11_ROWCOUNT, TABLE_ID_12_ROWCOUNT, TABLE_ID_13_ROWCOUNT, TABLE_ID_14_ROWCOUNT, TABLE_ID_15_ROWCOUNT;
}
+
transient protected boolean autoDelete = false;
private void commit() throws IOException {
- if (!fs.rename(outPath, finalPath)) {
- throw new IOException ("Unable to rename output to: " + finalPath);
+ if (!fs.rename(outPath, finalPath)) {
+ throw new IOException("Unable to rename output to: " + finalPath);
}
LOG.info("Committed to output file: " + finalPath);
}
+ @Override
protected void initializeOp(Configuration hconf) throws HiveException {
try {
- serializer = (Serializer)conf.getTableInfo().getDeserializerClass().newInstance();
+ serializer = (Serializer) conf.getTableInfo().getDeserializerClass()
+ .newInstance();
serializer.initialize(null, conf.getTableInfo().getProperties());
-
-
+
JobConf jc;
- if(hconf instanceof JobConf) {
- jc = (JobConf)hconf;
+ if (hconf instanceof JobConf) {
+ jc = (JobConf) hconf;
} else {
// test code path
jc = new JobConf(hconf, ExecDriver.class);
}
int id = conf.getDestTableId();
- if ((id != 0) && (id <= TableIdEnum.values().length)){
- String enumName = "TABLE_ID_"+String.valueOf(id)+"_ROWCOUNT";
+ if ((id != 0) && (id <= TableIdEnum.values().length)) {
+ String enumName = "TABLE_ID_" + String.valueOf(id) + "_ROWCOUNT";
tabIdEnum = TableIdEnum.valueOf(enumName);
row_count = new LongWritable();
statsMap.put(tabIdEnum, row_count);
-
+
}
String specPath = conf.getDirName();
Path tmpPath = Utilities.toTempPath(specPath);
- String taskId = Utilities.getTaskId(hconf);
- fs =(new Path(specPath)).getFileSystem(hconf);
+ String taskId = Utilities.getTaskId(hconf);
+ fs = (new Path(specPath)).getFileSystem(hconf);
finalPath = new Path(tmpPath, taskId);
outPath = new Path(tmpPath, Utilities.toTempPath(taskId));
LOG.info("Writing to temp file: FS " + outPath);
- HiveOutputFormat<?, ?> hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance();
+ HiveOutputFormat<?, ?> hiveOutputFormat = conf.getTableInfo()
+ .getOutputFileFormatClass().newInstance();
boolean isCompressed = conf.getCompressed();
// The reason to keep these instead of using
@@ -110,12 +116,16 @@
// getRecordWriter does not give us enough control over the file name that
// we create.
Path parent = Utilities.toTempPath(specPath);
- finalPath = HiveFileFormatUtils.getOutputFormatFinalPath(parent, jc, hiveOutputFormat, isCompressed, finalPath);
- final Class<? extends Writable> outputClass = serializer.getSerializedClass();
- outWriter = HiveFileFormatUtils.getHiveRecordWriter(jc, conf.getTableInfo(), outputClass, conf, outPath);
+ finalPath = HiveFileFormatUtils.getOutputFormatFinalPath(parent, jc,
+ hiveOutputFormat, isCompressed, finalPath);
+ final Class<? extends Writable> outputClass = serializer
+ .getSerializedClass();
+ outWriter = HiveFileFormatUtils.getHiveRecordWriter(jc, conf
+ .getTableInfo(), outputClass, conf, outPath);
// in recent hadoop versions, use deleteOnExit to clean tmp files.
- autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs, outPath);
+ autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs,
+ outPath);
initializeChildren(hconf);
} catch (HiveException e) {
@@ -126,31 +136,35 @@
}
}
- Writable recordValue;
+ Writable recordValue;
+
+ @Override
public void processOp(Object row, int tag) throws HiveException {
- // Since File Sink is a terminal operator, forward is not called - so, maintain the number of output rows explicitly
+ // Since File Sink is a terminal operator, forward is not called - so,
+ // maintain the number of output rows explicitly
if (counterNameToEnum != null) {
- ++this.outputRows;
- if (this.outputRows % 1000 == 0) {
+ ++outputRows;
+ if (outputRows % 1000 == 0) {
incrCounter(numOutputRowsCntr, outputRows);
- this.outputRows = 0;
+ outputRows = 0;
}
}
try {
- if (reporter != null)
+ if (reporter != null) {
reporter.progress();
+ }
// user SerDe to serialize r, and write it out
recordValue = serializer.serialize(row, inputObjInspectors[tag]);
- if (row_count != null){
- row_count.set(row_count.get()+ 1);
+ if (row_count != null) {
+ row_count.set(row_count.get() + 1);
}
-
+
outWriter.write(recordValue);
} catch (IOException e) {
- throw new HiveException (e);
+ throw new HiveException(e);
} catch (SerDeException e) {
- throw new HiveException (e);
+ throw new HiveException(e);
}
}
@@ -168,11 +182,13 @@
}
} else {
// Will come here if an Exception was thrown in map() or reduce().
- // Hadoop always call close() even if an Exception was thrown in map() or reduce().
+ // Hadoop always call close() even if an Exception was thrown in map() or
+ // reduce().
try {
outWriter.close(abort);
- if(!autoDelete)
+ if (!autoDelete) {
fs.delete(outPath, true);
+ }
} catch (Exception e) {
e.printStackTrace();
}
@@ -182,24 +198,27 @@
/**
* @return the name of the operator
*/
+ @Override
public String getName() {
return new String("FS");
}
@Override
- public void jobClose(Configuration hconf, boolean success) throws HiveException {
+ public void jobClose(Configuration hconf, boolean success)
+ throws HiveException {
try {
- if(conf != null) {
+ if (conf != null) {
String specPath = conf.getDirName();
FileSinkOperator.mvFileToFinalPath(specPath, hconf, success, LOG);
}
} catch (IOException e) {
- throw new HiveException (e);
+ throw new HiveException(e);
}
super.jobClose(hconf, success);
}
-
- public static void mvFileToFinalPath(String specPath, Configuration hconf, boolean success, Log LOG) throws IOException, HiveException{
+
+ public static void mvFileToFinalPath(String specPath, Configuration hconf,
+ boolean success, Log LOG) throws IOException, HiveException {
FileSystem fs = (new Path(specPath)).getFileSystem(hconf);
Path tmpPath = Utilities.toTempPath(specPath);
Path intermediatePath = new Path(tmpPath.getParent(), tmpPath.getName()
@@ -215,15 +234,15 @@
// Step2: remove any tmp file or double-committed output files
Utilities.removeTempOrDuplicateFiles(fs, intermediatePath);
// Step3: move to the file destination
- LOG.info("Moving tmp dir: " + intermediatePath + " to: "
- + finalPath);
+ LOG.info("Moving tmp dir: " + intermediatePath + " to: " + finalPath);
Utilities.renameOrMoveFiles(fs, intermediatePath, finalPath);
}
} else {
fs.delete(tmpPath, true);
}
}
-
+
+ @Override
public int getType() {
return OperatorType.FILESINK;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Thu Jan 21 10:37:58 2010
@@ -32,27 +32,34 @@
/**
* Filter operator implementation
**/
-public class FilterOperator extends Operator <filterDesc> implements Serializable {
+public class FilterOperator extends Operator<filterDesc> implements
+ Serializable {
private static final long serialVersionUID = 1L;
- public static enum Counter {FILTERED, PASSED}
+
+ public static enum Counter {
+ FILTERED, PASSED
+ }
+
transient private final LongWritable filtered_count, passed_count;
transient private ExprNodeEvaluator conditionEvaluator;
- transient private PrimitiveObjectInspector conditionInspector;
+ transient private PrimitiveObjectInspector conditionInspector;
transient private int consecutiveFails;
- transient int heartbeatInterval;
-
- public FilterOperator () {
+ transient int heartbeatInterval;
+
+ public FilterOperator() {
super();
filtered_count = new LongWritable();
passed_count = new LongWritable();
consecutiveFails = 0;
}
+ @Override
protected void initializeOp(Configuration hconf) throws HiveException {
try {
- heartbeatInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVESENDHEARTBEAT);
- this.conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
+ heartbeatInterval = HiveConf.getIntVar(hconf,
+ HiveConf.ConfVars.HIVESENDHEARTBEAT);
+ conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
statsMap.put(Counter.FILTERED, filtered_count);
statsMap.put(Counter.PASSED, passed_count);
conditionInspector = null;
@@ -62,24 +69,29 @@
initializeChildren(hconf);
}
+ @Override
public void processOp(Object row, int tag) throws HiveException {
ObjectInspector rowInspector = inputObjInspectors[tag];
if (conditionInspector == null) {
- conditionInspector = (PrimitiveObjectInspector)conditionEvaluator.initialize(rowInspector);
+ conditionInspector = (PrimitiveObjectInspector) conditionEvaluator
+ .initialize(rowInspector);
}
Object condition = conditionEvaluator.evaluate(row);
- Boolean ret = (Boolean)conditionInspector.getPrimitiveJavaObject(condition);
+ Boolean ret = (Boolean) conditionInspector
+ .getPrimitiveJavaObject(condition);
if (Boolean.TRUE.equals(ret)) {
forward(row, rowInspector);
- passed_count.set(passed_count.get()+1);
+ passed_count.set(passed_count.get() + 1);
consecutiveFails = 0;
} else {
- filtered_count.set(filtered_count.get()+1);
+ filtered_count.set(filtered_count.get() + 1);
consecutiveFails++;
-
- // In case of a lot of consecutive failures, send a heartbeat in order to avoid timeout
- if (((consecutiveFails % heartbeatInterval) == 0) && (reporter != null))
+
+ // In case of a lot of consecutive failures, send a heartbeat in order to
+ // avoid timeout
+ if (((consecutiveFails % heartbeatInterval) == 0) && (reporter != null)) {
reporter.progress();
+ }
}
}
@@ -91,6 +103,7 @@
return new String("FIL");
}
+ @Override
public int getType() {
return OperatorType.FILTER;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java Thu Jan 21 10:37:58 2010
@@ -25,18 +25,18 @@
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
/**
- * Forward Operator
- * Just forwards. Doesn't do anything itself.
+ * Forward Operator Just forwards. Doesn't do anything itself.
**/
-public class ForwardOperator extends Operator<forwardDesc> implements Serializable {
+public class ForwardOperator extends Operator<forwardDesc> implements
+ Serializable {
private static final long serialVersionUID = 1L;
@Override
- public void processOp(Object row, int tag)
- throws HiveException {
- forward(row, inputObjInspectors[tag]);
+ public void processOp(Object row, int tag) throws HiveException {
+ forward(row, inputObjInspectors[tag]);
}
-
+
+ @Override
public int getType() {
return OperatorType.FORWARD;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java Thu Jan 21 10:37:58 2010
@@ -25,37 +25,40 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
public class FunctionInfo {
-
- private boolean isNative;
- private String displayName;
+ private final boolean isNative;
+
+ private final String displayName;
private GenericUDF genericUDF;
private GenericUDTF genericUDTF;
-
+
private GenericUDAFResolver genericUDAFResolver;
-
- public FunctionInfo(boolean isNative, String displayName, GenericUDF genericUDF) {
+
+ public FunctionInfo(boolean isNative, String displayName,
+ GenericUDF genericUDF) {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDF = genericUDF;
}
- public FunctionInfo(boolean isNative, String displayName, GenericUDAFResolver genericUDAFResolver) {
+ public FunctionInfo(boolean isNative, String displayName,
+ GenericUDAFResolver genericUDAFResolver) {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDAFResolver = genericUDAFResolver;
}
- public FunctionInfo(boolean isNative, String displayName, GenericUDTF genericUDTF) {
+ public FunctionInfo(boolean isNative, String displayName,
+ GenericUDTF genericUDTF) {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDTF = genericUDTF;
}
-
+
/**
- * Get a new GenericUDF object for the function.
+ * Get a new GenericUDF object for the function.
*/
public GenericUDF getGenericUDF() {
// GenericUDF is stateful - we have to make a copy here
@@ -64,9 +67,9 @@
}
return FunctionRegistry.cloneGenericUDF(genericUDF);
}
-
+
/**
- * Get a new GenericUDTF object for the function.
+ * Get a new GenericUDTF object for the function.
*/
public GenericUDTF getGenericUDTF() {
// GenericUDTF is stateful too, copy
@@ -75,27 +78,27 @@
}
return FunctionRegistry.cloneGenericUDTF(genericUDTF);
}
-
+
/**
- * Get the GenericUDAFResolver object for the function.
+ * Get the GenericUDAFResolver object for the function.
*/
public GenericUDAFResolver getGenericUDAFResolver() {
return genericUDAFResolver;
}
-
+
/**
* Get the Class of the UDF
*/
public Class<?> getFunctionClass() {
if (isGenericUDF()) {
if (genericUDF instanceof GenericUDFBridge) {
- return ((GenericUDFBridge)genericUDF).getUdfClass();
+ return ((GenericUDFBridge) genericUDF).getUdfClass();
} else {
return genericUDF.getClass();
}
} else if (isGenericUDAF()) {
if (genericUDAFResolver instanceof GenericUDAFBridge) {
- return ((GenericUDAFBridge)genericUDAFResolver).getUDAFClass();
+ return ((GenericUDAFBridge) genericUDAFResolver).getUDAFClass();
} else {
return genericUDAFResolver.getClass();
}
@@ -104,39 +107,39 @@
}
return null;
}
-
+
/**
- * Get the display name for this function.
- * This should be transfered into exprNodeGenericUDFDesc, and will be
- * used as the first parameter to GenericUDF.getDisplayName() call, instead
- * of hard-coding the function name. This will solve the problem of
- * displaying only one name when a udf is registered under 2 names.
+ * Get the display name for this function. This should be transfered into
+ * exprNodeGenericUDFDesc, and will be used as the first parameter to
+ * GenericUDF.getDisplayName() call, instead of hard-coding the function name.
+ * This will solve the problem of displaying only one name when a udf is
+ * registered under 2 names.
*/
public String getDisplayName() {
return displayName;
}
-
+
/**
* Native functions cannot be unregistered.
*/
public boolean isNative() {
return isNative;
}
-
+
/**
* @return TRUE if the function is a GenericUDF
*/
public boolean isGenericUDF() {
return null != genericUDF;
}
-
+
/**
* @return TRUE if the function is a GenericUDAF
*/
public boolean isGenericUDAF() {
return null != genericUDAFResolver;
}
-
+
/**
* @return TRUE if the function is a GenericUDTF
*/
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Thu Jan 21 10:37:58 2010
@@ -18,9 +18,6 @@
package org.apache.hadoop.hive.ql.exec;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,6 +31,8 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
@@ -129,10 +128,10 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStdSample;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFVariance;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFVarianceSample;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -153,8 +152,8 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -167,7 +166,8 @@
public class FunctionRegistry {
- private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.exec.FunctionRegistry");
+ private static Log LOG = LogFactory
+ .getLog("org.apache.hadoop.hive.ql.exec.FunctionRegistry");
/**
* The mapping from expression function names to expression classes.
@@ -182,7 +182,7 @@
registerUDF("ascii", UDFAscii.class, false);
registerUDF("lpad", UDFLpad.class, false);
registerUDF("rpad", UDFRpad.class, false);
-
+
registerGenericUDF("size", GenericUDFSize.class);
registerUDF("round", UDFRound.class, false);
@@ -196,10 +196,10 @@
registerUDF("ln", UDFLn.class, false);
registerUDF("log2", UDFLog2.class, false);
- registerUDF("sin",UDFSin.class, false);
- registerUDF("asin",UDFAsin.class, false);
- registerUDF("cos",UDFCos.class, false);
- registerUDF("acos",UDFAcos.class, false);
+ registerUDF("sin", UDFSin.class, false);
+ registerUDF("asin", UDFAsin.class, false);
+ registerUDF("cos", UDFCos.class, false);
+ registerUDF("acos", UDFAcos.class, false);
registerUDF("log10", UDFLog10.class, false);
registerUDF("log", UDFLog.class, false);
registerUDF("exp", UDFExp.class, false);
@@ -210,7 +210,7 @@
registerUDF("bin", UDFBin.class, false);
registerUDF("hex", UDFHex.class, false);
registerUDF("unhex", UDFUnhex.class, false);
-
+
registerUDF("upper", UDFUpper.class, false);
registerUDF("lower", UDFLower.class, false);
registerUDF("ucase", UDFUpper.class, false);
@@ -222,7 +222,7 @@
registerUDF("reverse", UDFReverse.class, false);
registerGenericUDF("field", GenericUDFField.class);
registerUDF("find_in_set", UDFFindInSet.class, false);
-
+
registerUDF("like", UDFLike.class, true);
registerUDF("rlike", UDFRegExp.class, true);
registerUDF("regexp", UDFRegExp.class, true);
@@ -245,7 +245,7 @@
registerUDF("unix_timestamp", UDFUnixTimeStamp.class, false);
registerUDF("to_date", UDFDate.class, false);
registerUDF("weekofyear", UDFWeekOfYear.class, false);
-
+
registerUDF("date_add", UDFDateAdd.class, false);
registerUDF("date_sub", UDFDateSub.class, false);
registerUDF("datediff", UDFDateDiff.class, false);
@@ -282,31 +282,31 @@
registerGenericUDF("isnotnull", GenericUDFOPNotNull.class);
registerGenericUDF("if", GenericUDFIf.class);
-
+
// Aliases for Java Class Names
// These are used in getImplicitConvertUDFMethod
registerUDF(Constants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, false,
- UDFToBoolean.class.getSimpleName());
+ UDFToBoolean.class.getSimpleName());
registerUDF(Constants.TINYINT_TYPE_NAME, UDFToByte.class, false,
- UDFToByte.class.getSimpleName());
+ UDFToByte.class.getSimpleName());
registerUDF(Constants.SMALLINT_TYPE_NAME, UDFToShort.class, false,
- UDFToShort.class.getSimpleName());
+ UDFToShort.class.getSimpleName());
registerUDF(Constants.INT_TYPE_NAME, UDFToInteger.class, false,
- UDFToInteger.class.getSimpleName());
+ UDFToInteger.class.getSimpleName());
registerUDF(Constants.BIGINT_TYPE_NAME, UDFToLong.class, false,
- UDFToLong.class.getSimpleName());
+ UDFToLong.class.getSimpleName());
registerUDF(Constants.FLOAT_TYPE_NAME, UDFToFloat.class, false,
- UDFToFloat.class.getSimpleName());
+ UDFToFloat.class.getSimpleName());
registerUDF(Constants.DOUBLE_TYPE_NAME, UDFToDouble.class, false,
- UDFToDouble.class.getSimpleName());
+ UDFToDouble.class.getSimpleName());
registerUDF(Constants.STRING_TYPE_NAME, UDFToString.class, false,
- UDFToString.class.getSimpleName());
+ UDFToString.class.getSimpleName());
// Aggregate functions
registerGenericUDAF("sum", new GenericUDAFSum());
registerGenericUDAF("count", new GenericUDAFCount());
registerGenericUDAF("avg", new GenericUDAFAverage());
-
+
registerGenericUDAF("std", new GenericUDAFStd());
registerGenericUDAF("stddev", new GenericUDAFStd());
registerGenericUDAF("stddev_pop", new GenericUDAFStd());
@@ -314,10 +314,10 @@
registerGenericUDAF("variance", new GenericUDAFVariance());
registerGenericUDAF("var_pop", new GenericUDAFVariance());
registerGenericUDAF("var_samp", new GenericUDAFVarianceSample());
-
+
registerUDAF("max", UDAFMax.class);
registerUDAF("min", UDAFMin.class);
-
+
// Generic UDFs
registerGenericUDF("array", GenericUDFArray.class);
registerGenericUDF("map", GenericUDFMap.class);
@@ -331,98 +331,110 @@
registerGenericUDF("locate", GenericUDFLocate.class);
registerGenericUDF("elt", GenericUDFElt.class);
registerGenericUDF("concat_ws", GenericUDFConcatWS.class);
-
+
// Generic UDTF's
registerGenericUDTF("explode", GenericUDTFExplode.class);
}
- public static void registerTemporaryUDF(String functionName, Class<? extends UDF> UDFClass,
- boolean isOperator) {
+ public static void registerTemporaryUDF(String functionName,
+ Class<? extends UDF> UDFClass, boolean isOperator) {
registerUDF(false, functionName, UDFClass, isOperator);
}
static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
- boolean isOperator) {
+ boolean isOperator) {
registerUDF(true, functionName, UDFClass, isOperator);
}
- public static void registerUDF(boolean isNative, String functionName, Class<? extends UDF> UDFClass,
- boolean isOperator) {
- registerUDF(isNative, functionName, UDFClass, isOperator, functionName.toLowerCase());
+ public static void registerUDF(boolean isNative, String functionName,
+ Class<? extends UDF> UDFClass, boolean isOperator) {
+ registerUDF(isNative, functionName, UDFClass, isOperator, functionName
+ .toLowerCase());
}
- public static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
- boolean isOperator, String displayName) {
+ public static void registerUDF(String functionName,
+ Class<? extends UDF> UDFClass, boolean isOperator, String displayName) {
registerUDF(true, functionName, UDFClass, isOperator, displayName);
}
-
- public static void registerUDF(boolean isNative, String functionName, Class<? extends UDF> UDFClass,
- boolean isOperator, String displayName) {
+
+ public static void registerUDF(boolean isNative, String functionName,
+ Class<? extends UDF> UDFClass, boolean isOperator, String displayName) {
if (UDF.class.isAssignableFrom(UDFClass)) {
- FunctionInfo fI = new FunctionInfo(isNative, displayName,
+ FunctionInfo fI = new FunctionInfo(isNative, displayName,
new GenericUDFBridge(displayName, isOperator, UDFClass));
mFunctions.put(functionName.toLowerCase(), fI);
} else {
- throw new RuntimeException("Registering UDF Class " + UDFClass + " which does not extend " + UDF.class);
+ throw new RuntimeException("Registering UDF Class " + UDFClass
+ + " which does not extend " + UDF.class);
}
}
- public static void registerTemporaryGenericUDF(String functionName, Class<? extends GenericUDF> genericUDFClass) {
+ public static void registerTemporaryGenericUDF(String functionName,
+ Class<? extends GenericUDF> genericUDFClass) {
registerGenericUDF(false, functionName, genericUDFClass);
}
- static void registerGenericUDF(String functionName, Class<? extends GenericUDF> genericUDFClass) {
+ static void registerGenericUDF(String functionName,
+ Class<? extends GenericUDF> genericUDFClass) {
registerGenericUDF(true, functionName, genericUDFClass);
}
- public static void registerGenericUDF(boolean isNative, String functionName, Class<? extends GenericUDF> genericUDFClass) {
+ public static void registerGenericUDF(boolean isNative, String functionName,
+ Class<? extends GenericUDF> genericUDFClass) {
if (GenericUDF.class.isAssignableFrom(genericUDFClass)) {
- FunctionInfo fI = new FunctionInfo(isNative, functionName,
- (GenericUDF)ReflectionUtils.newInstance(genericUDFClass, null));
+ FunctionInfo fI = new FunctionInfo(isNative, functionName,
+ (GenericUDF) ReflectionUtils.newInstance(genericUDFClass, null));
mFunctions.put(functionName.toLowerCase(), fI);
} else {
- throw new RuntimeException("Registering GenericUDF Class " + genericUDFClass
- + " which does not extend " + GenericUDF.class);
+ throw new RuntimeException("Registering GenericUDF Class "
+ + genericUDFClass + " which does not extend " + GenericUDF.class);
}
}
- public static void registerTemporaryGenericUDTF(String functionName, Class<? extends GenericUDTF> genericUDTFClass) {
+ public static void registerTemporaryGenericUDTF(String functionName,
+ Class<? extends GenericUDTF> genericUDTFClass) {
registerGenericUDTF(false, functionName, genericUDTFClass);
}
- static void registerGenericUDTF(String functionName, Class<? extends GenericUDTF> genericUDTFClass) {
+
+ static void registerGenericUDTF(String functionName,
+ Class<? extends GenericUDTF> genericUDTFClass) {
registerGenericUDTF(true, functionName, genericUDTFClass);
}
- public static void registerGenericUDTF(boolean isNative, String functionName, Class<? extends GenericUDTF> genericUDTFClass) {
+ public static void registerGenericUDTF(boolean isNative, String functionName,
+ Class<? extends GenericUDTF> genericUDTFClass) {
if (GenericUDTF.class.isAssignableFrom(genericUDTFClass)) {
- FunctionInfo fI = new FunctionInfo(isNative, functionName,
- (GenericUDTF)ReflectionUtils.newInstance(genericUDTFClass, null));
+ FunctionInfo fI = new FunctionInfo(isNative, functionName,
+ (GenericUDTF) ReflectionUtils.newInstance(genericUDTFClass, null));
mFunctions.put(functionName.toLowerCase(), fI);
} else {
- throw new RuntimeException("Registering GenericUDTF Class " + genericUDTFClass
- + " which does not extend " + GenericUDTF.class);
+ throw new RuntimeException("Registering GenericUDTF Class "
+ + genericUDTFClass + " which does not extend " + GenericUDTF.class);
}
}
-
+
public static FunctionInfo getFunctionInfo(String functionName) {
return mFunctions.get(functionName.toLowerCase());
}
/**
- * Returns a set of registered function names.
- * This is used for the CLI command "SHOW FUNCTIONS;"
- * @return set of strings contains function names
+ * Returns a set of registered function names. This is used for the CLI
+ * command "SHOW FUNCTIONS;"
+ *
+ * @return set of strings contains function names
*/
public static Set<String> getFunctionNames() {
return mFunctions.keySet();
}
/**
- * Returns a set of registered function names.
- * This is used for the CLI command "SHOW FUNCTIONS 'regular expression';"
- * Returns an empty set when the regular expression is not valid.
- * @param funcPatternStr regular expression of the interested function names
- * @return set of strings contains function names
+ * Returns a set of registered function names. This is used for the CLI
+ * command "SHOW FUNCTIONS 'regular expression';" Returns an empty set when
+ * the regular expression is not valid.
+ *
+ * @param funcPatternStr
+ * regular expression of the interested function names
+ * @return set of strings contains function names
*/
public static Set<String> getFunctionNames(String funcPatternStr) {
Set<String> funcNames = new TreeSet<String>();
@@ -442,17 +454,19 @@
/**
* Returns the set of synonyms of the supplied function.
- * @param funcName the name of the function
+ *
+ * @param funcName
+ * the name of the function
* @return Set of synonyms for funcName
*/
public static Set<String> getFunctionSynonyms(String funcName) {
Set<String> synonyms = new HashSet<String>();
-
+
FunctionInfo funcInfo = getFunctionInfo(funcName);
if (null == funcInfo) {
return synonyms;
}
-
+
Class<?> funcClass = funcInfo.getFunctionClass();
for (String name : mFunctions.keySet()) {
if (name.equals(funcName)) {
@@ -462,18 +476,19 @@
synonyms.add(name);
}
}
-
+
return synonyms;
}
-
static Map<TypeInfo, Integer> numericTypes = new HashMap<TypeInfo, Integer>();
static List<TypeInfo> numericTypeList = new ArrayList<TypeInfo>();
+
static void registerNumericType(String typeName, int level) {
TypeInfo t = TypeInfoFactory.getPrimitiveTypeInfo(typeName);
numericTypeList.add(t);
- numericTypes.put(t, level);
+ numericTypes.put(t, level);
}
+
static {
registerNumericType(Constants.TINYINT_TYPE_NAME, 1);
registerNumericType(Constants.SMALLINT_TYPE_NAME, 2);
@@ -485,21 +500,23 @@
}
/**
- * Find a common class that objects of both TypeInfo a and TypeInfo b can convert to.
- * This is used for comparing objects of type a and type b.
+ * Find a common class that objects of both TypeInfo a and TypeInfo b can
+ * convert to. This is used for comparing objects of type a and type b.
*
- * When we are comparing string and double, we will always convert both of them
- * to double and then compare.
+ * When we are comparing string and double, we will always convert both of
+ * them to double and then compare.
*
* @return null if no common class could be found.
*/
public static TypeInfo getCommonClassForComparison(TypeInfo a, TypeInfo b) {
// If same return one of them
- if (a.equals(b)) return a;
-
- for (TypeInfo t: numericTypeList) {
- if (FunctionRegistry.implicitConvertable(a, t) &&
- FunctionRegistry.implicitConvertable(b, t)) {
+ if (a.equals(b)) {
+ return a;
+ }
+
+ for (TypeInfo t : numericTypeList) {
+ if (FunctionRegistry.implicitConvertable(a, t)
+ && FunctionRegistry.implicitConvertable(b, t)) {
return t;
}
}
@@ -507,8 +524,8 @@
}
/**
- * Find a common class that objects of both TypeInfo a and TypeInfo b can convert to.
- * This is used for places other than comparison.
+ * Find a common class that objects of both TypeInfo a and TypeInfo b can
+ * convert to. This is used for places other than comparison.
*
* The common class of string and double is string.
*
@@ -524,7 +541,9 @@
return (ai > bi) ? a : b;
}
- /** Returns whether it is possible to implicitly convert an object of Class from to Class to.
+ /**
+ * Returns whether it is possible to implicitly convert an object of Class
+ * from to Class to.
*/
public static boolean implicitConvertable(TypeInfo from, TypeInfo to) {
if (from.equals(to)) {
@@ -540,42 +559,52 @@
return true;
}
- // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double -> String
+ // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double
+ // -> String
Integer f = numericTypes.get(from);
Integer t = numericTypes.get(to);
- if (f == null || t == null) return false;
- if (f.intValue() > t.intValue()) return false;
+ if (f == null || t == null) {
+ return false;
+ }
+ if (f.intValue() > t.intValue()) {
+ return false;
+ }
return true;
}
/**
* Get the GenericUDAF evaluator for the name and argumentClasses.
- * @param name the name of the UDAF
+ *
+ * @param name
+ * the name of the UDAF
* @param argumentTypeInfos
* @return The UDAF evaluator
*/
- public static GenericUDAFEvaluator getGenericUDAFEvaluator(String name, List<TypeInfo> argumentTypeInfos)
- throws SemanticException {
+ public static GenericUDAFEvaluator getGenericUDAFEvaluator(String name,
+ List<TypeInfo> argumentTypeInfos) throws SemanticException {
GenericUDAFResolver udaf = getGenericUDAFResolver(name);
- if (udaf == null) return null;
+ if (udaf == null) {
+ return null;
+ }
TypeInfo[] parameters = new TypeInfo[argumentTypeInfos.size()];
- for(int i=0; i<parameters.length; i++) {
+ for (int i = 0; i < parameters.length; i++) {
parameters[i] = argumentTypeInfos.get(i);
}
return udaf.getEvaluator(parameters);
}
/**
- * This method is shared between UDFRegistry and UDAFRegistry.
- * methodName will be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial" for UDAFRegistry.
+ * This method is shared between UDFRegistry and UDAFRegistry. methodName will
+ * be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial"
+ * for UDAFRegistry.
*/
- public static <T> Method getMethodInternal(Class<? extends T> udfClass, String methodName, boolean exact,
- List<TypeInfo> argumentClasses) {
+ public static <T> Method getMethodInternal(Class<? extends T> udfClass,
+ String methodName, boolean exact, List<TypeInfo> argumentClasses) {
List<Method> mlist = new ArrayList<Method>();
- for(Method m: Arrays.asList(udfClass.getMethods())) {
+ for (Method m : Arrays.asList(udfClass.getMethods())) {
if (m.getName().equals(methodName)) {
mlist.add(m);
}
@@ -584,20 +613,24 @@
return getMethodInternal(mlist, exact, argumentClasses);
}
- public static void registerTemporaryGenericUDAF(String functionName, GenericUDAFResolver genericUDAFResolver) {
+ public static void registerTemporaryGenericUDAF(String functionName,
+ GenericUDAFResolver genericUDAFResolver) {
registerGenericUDAF(false, functionName, genericUDAFResolver);
}
- static void registerGenericUDAF(String functionName, GenericUDAFResolver genericUDAFResolver) {
+ static void registerGenericUDAF(String functionName,
+ GenericUDAFResolver genericUDAFResolver) {
registerGenericUDAF(true, functionName, genericUDAFResolver);
}
- public static void registerGenericUDAF(boolean isNative, String functionName, GenericUDAFResolver genericUDAFResolver) {
- mFunctions.put(functionName.toLowerCase(),
- new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver));
+ public static void registerGenericUDAF(boolean isNative, String functionName,
+ GenericUDAFResolver genericUDAFResolver) {
+ mFunctions.put(functionName.toLowerCase(), new FunctionInfo(isNative,
+ functionName.toLowerCase(), genericUDAFResolver));
}
- public static void registerTemporaryUDAF(String functionName, Class<? extends UDAF> udafClass) {
+ public static void registerTemporaryUDAF(String functionName,
+ Class<? extends UDAF> udafClass) {
registerUDAF(false, functionName, udafClass);
}
@@ -605,20 +638,23 @@
registerUDAF(true, functionName, udafClass);
}
- public static void registerUDAF(boolean isNative, String functionName, Class<? extends UDAF> udafClass) {
- mFunctions.put(functionName.toLowerCase(),
- new FunctionInfo(isNative, functionName.toLowerCase(),
- new GenericUDAFBridge((UDAF)ReflectionUtils.newInstance(udafClass, null))));
+ public static void registerUDAF(boolean isNative, String functionName,
+ Class<? extends UDAF> udafClass) {
+ mFunctions.put(functionName.toLowerCase(), new FunctionInfo(isNative,
+ functionName.toLowerCase(), new GenericUDAFBridge(
+ (UDAF) ReflectionUtils.newInstance(udafClass, null))));
}
- public static void unregisterTemporaryUDF(String functionName) throws HiveException {
+ public static void unregisterTemporaryUDF(String functionName)
+ throws HiveException {
FunctionInfo fi = mFunctions.get(functionName.toLowerCase());
- if(fi != null) {
- if(!fi.isNative())
+ if (fi != null) {
+ if (!fi.isNative()) {
mFunctions.remove(functionName.toLowerCase());
- else
- throw new HiveException("Function " + functionName
+ } else {
+ throw new HiveException("Function " + functionName
+ " is hive native, it can't be dropped");
+ }
}
}
@@ -632,44 +668,47 @@
return result;
}
- public static Object invoke(Method m, Object thisObject, Object... arguments) throws HiveException {
+ public static Object invoke(Method m, Object thisObject, Object... arguments)
+ throws HiveException {
Object o;
try {
o = m.invoke(thisObject, arguments);
} catch (Exception e) {
- String thisObjectString = "" + thisObject + " of class " +
- (thisObject == null? "null" : thisObject.getClass().getName());
+ String thisObjectString = "" + thisObject + " of class "
+ + (thisObject == null ? "null" : thisObject.getClass().getName());
StringBuilder argumentString = new StringBuilder();
if (arguments == null) {
argumentString.append("null");
} else {
argumentString.append("{");
- for (int i=0; i<arguments.length; i++) {
- if (i>0) {
+ for (int i = 0; i < arguments.length; i++) {
+ if (i > 0) {
argumentString.append(", ");
}
if (arguments[i] == null) {
argumentString.append("null");
} else {
- argumentString.append("" + arguments[i] + ":" + arguments[i].getClass().getName());
+ argumentString.append("" + arguments[i] + ":"
+ + arguments[i].getClass().getName());
}
}
argumentString.append("} of size " + arguments.length);
}
throw new HiveException("Unable to execute method " + m + " "
- + " on object " + thisObjectString
- + " with arguments " + argumentString.toString(), e);
+ + " on object " + thisObjectString + " with arguments "
+ + argumentString.toString(), e);
}
return o;
}
/**
- * Returns -1 if passed does not match accepted.
- * Otherwise return the cost (usually 0 for no conversion and 1 for conversion).
+ * Returns -1 if passed does not match accepted. Otherwise return the cost
+ * (usually 0 for no conversion and 1 for conversion).
*/
- public static int matchCost(TypeInfo argumentPassed, TypeInfo argumentAccepted, boolean exact) {
+ public static int matchCost(TypeInfo argumentPassed,
+ TypeInfo argumentAccepted, boolean exact) {
if (argumentAccepted.equals(argumentPassed)) {
// matches
return 0;
@@ -678,26 +717,34 @@
// passing null matches everything
return 0;
}
- if (argumentPassed.getCategory().equals(Category.LIST)
+ if (argumentPassed.getCategory().equals(Category.LIST)
&& argumentAccepted.getCategory().equals(Category.LIST)) {
// lists are compatible if and only-if the elements are compatible
- TypeInfo argumentPassedElement = ((ListTypeInfo)argumentPassed).getListElementTypeInfo();
- TypeInfo argumentAcceptedElement = ((ListTypeInfo)argumentAccepted).getListElementTypeInfo();
+ TypeInfo argumentPassedElement = ((ListTypeInfo) argumentPassed)
+ .getListElementTypeInfo();
+ TypeInfo argumentAcceptedElement = ((ListTypeInfo) argumentAccepted)
+ .getListElementTypeInfo();
return matchCost(argumentPassedElement, argumentAcceptedElement, exact);
}
- if (argumentPassed.getCategory().equals(Category.MAP)
+ if (argumentPassed.getCategory().equals(Category.MAP)
&& argumentAccepted.getCategory().equals(Category.MAP)) {
// lists are compatible if and only-if the elements are compatible
- TypeInfo argumentPassedKey = ((MapTypeInfo)argumentPassed).getMapKeyTypeInfo();
- TypeInfo argumentAcceptedKey = ((MapTypeInfo)argumentAccepted).getMapKeyTypeInfo();
- TypeInfo argumentPassedValue = ((MapTypeInfo)argumentPassed).getMapValueTypeInfo();
- TypeInfo argumentAcceptedValue = ((MapTypeInfo)argumentAccepted).getMapValueTypeInfo();
+ TypeInfo argumentPassedKey = ((MapTypeInfo) argumentPassed)
+ .getMapKeyTypeInfo();
+ TypeInfo argumentAcceptedKey = ((MapTypeInfo) argumentAccepted)
+ .getMapKeyTypeInfo();
+ TypeInfo argumentPassedValue = ((MapTypeInfo) argumentPassed)
+ .getMapValueTypeInfo();
+ TypeInfo argumentAcceptedValue = ((MapTypeInfo) argumentAccepted)
+ .getMapValueTypeInfo();
int cost1 = matchCost(argumentPassedKey, argumentAcceptedKey, exact);
int cost2 = matchCost(argumentPassedValue, argumentAcceptedValue, exact);
- if (cost1 < 0 || cost2 < 0) return -1;
+ if (cost1 < 0 || cost2 < 0) {
+ return -1;
+ }
return Math.max(cost1, cost2);
}
-
+
if (argumentAccepted.equals(TypeInfoFactory.unknownTypeInfo)) {
// accepting Object means accepting everything,
// but there is a conversion cost.
@@ -706,16 +753,20 @@
if (!exact && implicitConvertable(argumentPassed, argumentAccepted)) {
return 1;
}
-
+
return -1;
}
-
+
/**
- * Gets the closest matching method corresponding to the argument list from a list of methods.
- *
- * @param mlist The list of methods to inspect.
- * @param exact Boolean to indicate whether this is an exact match or not.
- * @param argumentsPassed The classes for the argument.
+ * Gets the closest matching method corresponding to the argument list from a
+ * list of methods.
+ *
+ * @param mlist
+ * The list of methods to inspect.
+ * @param exact
+ * Boolean to indicate whether this is an exact match or not.
+ * @param argumentsPassed
+ * The classes for the argument.
* @return The matching method.
*/
public static Method getMethodInternal(List<Method> mlist, boolean exact,
@@ -723,40 +774,45 @@
int leastConversionCost = Integer.MAX_VALUE;
Method udfMethod = null;
- for (Method m: mlist) {
+ for (Method m : mlist) {
List<TypeInfo> argumentsAccepted = TypeInfoUtils.getParameterTypeInfos(m,
argumentsPassed.size());
if (argumentsAccepted == null) {
// null means the method does not accept number of arguments passed.
continue;
}
-
+
boolean match = (argumentsAccepted.size() == argumentsPassed.size());
int conversionCost = 0;
- for(int i=0; i<argumentsPassed.size() && match; i++) {
- int cost = matchCost(argumentsPassed.get(i), argumentsAccepted.get(i), exact);
+ for (int i = 0; i < argumentsPassed.size() && match; i++) {
+ int cost = matchCost(argumentsPassed.get(i), argumentsAccepted.get(i),
+ exact);
if (cost == -1) {
match = false;
} else {
conversionCost += cost;
}
}
- LOG.debug("Method " + (match ? "did": "didn't") + " match: passed = " + argumentsPassed
- + " accepted = " + argumentsAccepted + " method = " + m);
+ LOG.debug("Method " + (match ? "did" : "didn't") + " match: passed = "
+ + argumentsPassed + " accepted = " + argumentsAccepted + " method = "
+ + m);
if (match) {
// Always choose the function with least implicit conversions.
if (conversionCost < leastConversionCost) {
udfMethod = m;
leastConversionCost = conversionCost;
// Found an exact match
- if (leastConversionCost == 0) break;
- } else if (conversionCost == leastConversionCost){
- // Ambiguous call: two methods with the same number of implicit conversions
+ if (leastConversionCost == 0) {
+ break;
+ }
+ } else if (conversionCost == leastConversionCost) {
+ // Ambiguous call: two methods with the same number of implicit
+ // conversions
LOG.info("Ambigious methods: passed = " + argumentsPassed
+ " method 1 = " + udfMethod + " method 2 = " + m);
udfMethod = null;
- // Don't break! We might find a better match later.
+ // Don't break! We might find a better match later.
} else {
// do nothing if implicitConversions > leastImplicitConversions
}
@@ -764,10 +820,10 @@
}
return udfMethod;
}
-
+
/**
- * A shortcut to get the "index" GenericUDF.
- * This is used for getting elements out of array and getting values out of map.
+ * A shortcut to get the "index" GenericUDF. This is used for getting elements
+ * out of array and getting values out of map.
*/
public static GenericUDF getGenericUDFForIndex() {
return FunctionRegistry.getFunctionInfo("index").getGenericUDF();
@@ -779,7 +835,7 @@
public static GenericUDF getGenericUDFForAnd() {
return FunctionRegistry.getFunctionInfo("and").getGenericUDF();
}
-
+
/**
* Create a copy of an existing GenericUDF.
*/
@@ -787,13 +843,15 @@
if (null == genericUDF) {
return null;
}
-
+
if (genericUDF instanceof GenericUDFBridge) {
- GenericUDFBridge bridge = (GenericUDFBridge)genericUDF;
- return new GenericUDFBridge(bridge.getUdfName(), bridge.isOperator(), bridge.getUdfClass());
+ GenericUDFBridge bridge = (GenericUDFBridge) genericUDF;
+ return new GenericUDFBridge(bridge.getUdfName(), bridge.isOperator(),
+ bridge.getUdfClass());
}
-
- return (GenericUDF)ReflectionUtils.newInstance(genericUDF.getClass(), null);
+
+ return (GenericUDF) ReflectionUtils
+ .newInstance(genericUDF.getClass(), null);
}
/**
@@ -803,25 +861,27 @@
if (null == genericUDTF) {
return null;
}
- return (GenericUDTF)ReflectionUtils.newInstance(genericUDTF.getClass(), null);
+ return (GenericUDTF) ReflectionUtils.newInstance(genericUDTF.getClass(),
+ null);
}
-
+
/**
- * Get the UDF class from an exprNodeDesc.
- * Returns null if the exprNodeDesc does not contain a UDF class.
+ * Get the UDF class from an exprNodeDesc. Returns null if the exprNodeDesc
+ * does not contain a UDF class.
*/
private static Class<? extends UDF> getUDFClassFromExprDesc(exprNodeDesc desc) {
if (!(desc instanceof exprNodeGenericFuncDesc)) {
return null;
}
- exprNodeGenericFuncDesc genericFuncDesc = (exprNodeGenericFuncDesc)desc;
+ exprNodeGenericFuncDesc genericFuncDesc = (exprNodeGenericFuncDesc) desc;
if (!(genericFuncDesc.getGenericUDF() instanceof GenericUDFBridge)) {
return null;
}
- GenericUDFBridge bridge = (GenericUDFBridge)(genericFuncDesc.getGenericUDF());
+ GenericUDFBridge bridge = (GenericUDFBridge) (genericFuncDesc
+ .getGenericUDF());
return bridge.getUdfClass();
}
-
+
/**
* Returns whether a GenericUDF is deterministic or not.
*/
@@ -830,42 +890,41 @@
if (genericUDFType != null && genericUDFType.deterministic() == false) {
return false;
}
-
+
if (genericUDF instanceof GenericUDFBridge) {
- GenericUDFBridge bridge = (GenericUDFBridge)(genericUDF);
+ GenericUDFBridge bridge = (GenericUDFBridge) (genericUDF);
UDFType bridgeUDFType = bridge.getUdfClass().getAnnotation(UDFType.class);
if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) {
return false;
}
}
-
+
return true;
}
-
+
/**
- * Returns whether the exprNodeDesc is a node of "and", "or", "not".
+ * Returns whether the exprNodeDesc is a node of "and", "or", "not".
*/
public static boolean isOpAndOrNot(exprNodeDesc desc) {
Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
- return UDFOPAnd.class == udfClass
- || UDFOPOr.class == udfClass
+ return UDFOPAnd.class == udfClass || UDFOPOr.class == udfClass
|| UDFOPNot.class == udfClass;
}
-
+
/**
- * Returns whether the exprNodeDesc is a node of "and".
+ * Returns whether the exprNodeDesc is a node of "and".
*/
public static boolean isOpAnd(exprNodeDesc desc) {
Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
return UDFOPAnd.class == udfClass;
}
-
+
/**
- * Returns whether the exprNodeDesc is a node of "positive".
+ * Returns whether the exprNodeDesc is a node of "positive".
*/
public static boolean isOpPositive(exprNodeDesc desc) {
Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
return UDFOPPositive.class == udfClass;
}
-
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=901644&r1=901643&r2=901644&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Thu Jan 21 10:37:58 2010
@@ -40,17 +40,17 @@
private static final Log LOG = LogFactory.getLog("hive.ql.exec.FunctionTask");
transient HiveConf conf;
-
+
public FunctionTask() {
super();
}
-
+
@Override
public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx) {
super.initialize(conf, queryPlan, ctx);
this.conf = conf;
}
-
+
@Override
public int execute() {
createFunctionDesc createFunctionDesc = work.getCreateFunctionDesc();
@@ -68,25 +68,26 @@
private int createFunction(createFunctionDesc createFunctionDesc) {
try {
Class<?> udfClass = getUdfClass(createFunctionDesc);
- if(UDF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryUDF(createFunctionDesc.getFunctionName(),
- (Class<? extends UDF>) udfClass, false);
+ if (UDF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryUDF(createFunctionDesc
+ .getFunctionName(), (Class<? extends UDF>) udfClass, false);
return 0;
- } else if(GenericUDF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDF(createFunctionDesc.getFunctionName(),
- (Class<? extends GenericUDF>) udfClass);
+ } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDF(createFunctionDesc
+ .getFunctionName(), (Class<? extends GenericUDF>) udfClass);
return 0;
- } else if(GenericUDTF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDTF(createFunctionDesc.getFunctionName(),
- (Class<? extends GenericUDTF>) udfClass);
+ } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDTF(createFunctionDesc
+ .getFunctionName(), (Class<? extends GenericUDTF>) udfClass);
return 0;
- } else if(UDAF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryUDAF(createFunctionDesc.getFunctionName(),
- (Class<? extends UDAF>) udfClass);
+ } else if (UDAF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryUDAF(createFunctionDesc
+ .getFunctionName(), (Class<? extends UDAF>) udfClass);
return 0;
- } else if(GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDAF(createFunctionDesc.getFunctionName(),
- (GenericUDAFResolver)ReflectionUtils.newInstance(udfClass, null));
+ } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDAF(createFunctionDesc
+ .getFunctionName(), (GenericUDAFResolver) ReflectionUtils
+ .newInstance(udfClass, null));
return 0;
}
return 1;
@@ -99,7 +100,8 @@
private int dropFunction(dropFunctionDesc dropFunctionDesc) {
try {
- FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc.getFunctionName());
+ FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc
+ .getFunctionName());
return 0;
} catch (HiveException e) {
LOG.info("drop function: " + StringUtils.stringifyException(e));
@@ -112,7 +114,8 @@
throws ClassNotFoundException {
return Class.forName(desc.getClassName(), true, JavaUtils.getClassLoader());
}
-
+
+ @Override
public int getType() {
return StageType.FUNC;
}