You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/09/02 18:30:34 UTC
svn commit: r1622060 [1/3] - in /hive/trunk:
beeline/src/java/org/apache/hive/beeline/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/
hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ hc...
Author: hashutosh
Date: Tue Sep 2 16:30:33 2014
New Revision: 1622060
URL: http://svn.apache.org/r1622060
Log:
HIVE-7931 : Convert all tabs to spaces [code cleanup] (Lars Francke via Ashutosh Chauhan)
Modified:
hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
hive/trunk/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IdentityExpression.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileRecordReader.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadMultiFilesDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCumeDist.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFDenseRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFNTile.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRowNumber.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseTrim.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBetween.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLpad.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRpad.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToBinary.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java Tue Sep 2 16:30:33 2014
@@ -287,7 +287,7 @@ public class HiveSchemaTool {
}
} catch (IOException e) {
throw new HiveMetaException("Schema initialization FAILED!" +
- " Metastore state would be inconsistent !!", e);
+ " Metastore state would be inconsistent !!", e);
}
}
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java Tue Sep 2 16:30:33 2014
@@ -116,8 +116,7 @@ public class StatsSetupConst {
/**
* @return List of all supported statistics
*/
- public static final String[] supportedStats = new String[]
- {NUM_FILES,ROW_COUNT,TOTAL_SIZE,RAW_DATA_SIZE};
+ public static final String[] supportedStats = {NUM_FILES,ROW_COUNT,TOTAL_SIZE,RAW_DATA_SIZE};
/**
* @return List of all statistics that need to be collected during query execution. These are
@@ -142,8 +141,8 @@ public class StatsSetupConst {
public static final String FALSE = "false";
- public static boolean areStatsUptoDate(Map<String,String> params) {
- String statsAcc = params.get(COLUMN_STATS_ACCURATE);
- return statsAcc == null ? false : statsAcc.equals(TRUE);
+ public static boolean areStatsUptoDate(Map<String, String> params) {
+ String statsAcc = params.get(COLUMN_STATS_ACCURATE);
+ return statsAcc == null ? false : statsAcc.equals(TRUE);
}
}
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Sep 2 16:30:33 2014
@@ -807,7 +807,7 @@ public class HiveConf extends Configurat
" config (hive.exec.orc.block.padding.tolerance)."),
HIVEMERGEINPUTFORMATSTRIPELEVEL("hive.merge.input.format.stripe.level",
"org.apache.hadoop.hive.ql.io.orc.OrcFileStripeMergeInputFormat",
- "Input file format to use for ORC stripe level merging (for internal use only)"),
+ "Input file format to use for ORC stripe level merging (for internal use only)"),
HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS(
"hive.merge.current.job.has.dynamic.partitions", false, ""),
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Sep 2 16:30:33 2014
@@ -653,7 +653,7 @@ public class Server {
verifyParam(inputs, "input");
verifyParam(mapper, "mapper");
verifyParam(reducer, "reducer");
-
+
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("input", inputs);
@@ -680,8 +680,8 @@ public class Server {
/**
* Run a MapReduce Jar job.
* Params correspond to the REST api params
- * @param usesHcatalog if {@code true}, means the Jar uses HCat and thus needs to access
- * metastore, which requires additional steps for WebHCat to perform in a secure cluster.
+ * @param usesHcatalog if {@code true}, means the Jar uses HCat and thus needs to access
+ * metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* @param callback URL which WebHCat will call when the hive job finishes
* @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
*/
@@ -703,7 +703,7 @@ public class Server {
verifyUser();
verifyParam(jar, "jar");
verifyParam(mainClass, "class");
-
+
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("jar", jar);
@@ -729,7 +729,7 @@ public class Server {
* Run a Pig job.
* Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usesHcatalog},
* is interpreted as true.
- * @param usesHcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
+ * @param usesHcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
* metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* This does nothing to ensure that Pig is installed on target node in the cluster.
* @param callback URL which WebHCat will call when the hive job finishes
@@ -752,7 +752,7 @@ public class Server {
if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
}
-
+
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
@@ -819,7 +819,7 @@ public class Server {
* @param execute SQL statement to run, equivalent to "-e" from hive command line
* @param srcFile name of hive script file to run, equivalent to "-f" from hive
* command line
- * @param hiveArgs additional command line argument passed to the hive command line.
+ * @param hiveArgs additional command line argument passed to the hive command line.
* Please check https://cwiki.apache.org/Hive/languagemanual-cli.html
* for detailed explanation of command line arguments
* @param otherFiles additional files to be shipped to the launcher, such as the jars
@@ -846,7 +846,7 @@ public class Server {
if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
}
-
+
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
@@ -903,42 +903,42 @@ public class Server {
* Example usages:
* 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan'
* Return all the Job IDs submitted by hsubramaniyan
- * 2. curl -s
+ * 2. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&showall=true'
* Return all the Job IDs that are visible to hsubramaniyan
* 3. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&jobid=job_201312091733_0003'
* Return all the Job IDs for hsubramaniyan after job_201312091733_0003.
- * 4. curl -s 'http://localhost:50111/templeton/v1/jobs?
+ * 4. curl -s 'http://localhost:50111/templeton/v1/jobs?
* user.name=hsubramaniyan&jobid=job_201312091733_0003&numrecords=5'
- * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003.
- * 5. curl -s
+ * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003.
+ * 5. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&numrecords=5'
- * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list
+ * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list
* lexicographically.
* </p>
* <p>
* Supporting pagination using "jobid" and "numrecords" parameters:
* Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n
- * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid"
- * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's
+ * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid"
+ * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's
* last record and get the Job Id of the last record as job_yyy_k, else quit.
* Step 4: set "jobid"=job_yyy_k and go to step 2.
- * </p>
+ * </p>
* @param fields If "fields" set to "*", the request will return full details of the job.
* If "fields" is missing, will only return the job ID. Currently the value can only
* be "*", other values are not allowed and will throw exception.
* @param showall If "showall" is set to "true", the request will return all jobs the user
* has permission to view, not only the jobs belonging to the user.
- * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater
- * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001",
- * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of
+ * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater
+ * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001",
+ * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of
* records returned depends on the value of "numrecords".
- * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords
- * records appearing after "jobid" will be returned after sorting the Job Id list
- * lexicographically.
- * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will
- * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present
+ * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords
+ * records appearing after "jobid" will be returned after sorting the Job Id list
+ * lexicographically.
+ * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will
+ * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present
* and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are returned.
* @return list of job items based on the filter conditions specified by the user.
*/
@@ -950,7 +950,7 @@ public class Server {
@QueryParam("jobid") String jobid,
@QueryParam("numrecords") String numrecords)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
-
+
verifyUser();
boolean showDetails = false;
@@ -971,9 +971,9 @@ public class Server {
try {
if (numrecords != null) {
numRecords = Integer.parseInt(numrecords);
- if (numRecords <= 0) {
- throw new BadParam("numrecords should be an integer > 0");
- }
+ if (numRecords <= 0) {
+ throw new BadParam("numrecords should be an integer > 0");
+ }
}
else {
numRecords = -1;
@@ -983,18 +983,18 @@ public class Server {
throw new BadParam("Invalid numrecords format: numrecords should be an integer > 0");
}
- // Sort the list lexicographically
+ // Sort the list lexicographically
Collections.sort(list);
for (String job : list) {
// If numRecords = -1, fetch all records.
// Hence skip all the below checks when numRecords = -1.
if (numRecords != -1) {
- // If currRecord >= numRecords, we have already fetched the top #numRecords
+ // If currRecord >= numRecords, we have already fetched the top #numRecords
if (currRecord >= numRecords) {
break;
- }
- // If the current record needs to be returned based on the
+ }
+ // If the current record needs to be returned based on the
// filter conditions specified by the user, increment the counter
else if ((jobid != null && job.compareTo(jobid) > 0) || jobid == null) {
currRecord++;
@@ -1101,7 +1101,7 @@ public class Server {
* value of user.name query param, in kerberos mode it's the kinit'ed user.
*/
private String getRequestingUser() {
- if (theSecurityContext == null) {
+ if (theSecurityContext == null) {
return null;
}
String userName = null;
@@ -1114,7 +1114,7 @@ public class Server {
if(userName == null) {
return null;
}
- //map hue/foo.bar@something.com->hue since user group checks
+ //map hue/foo.bar@something.com->hue since user group checks
// and config files are in terms of short name
return UserGroupInformation.createRemoteUser(userName).getShortUserName();
}
@@ -1161,7 +1161,7 @@ public class Server {
return unkHost;
}
}
-
+
private void checkEnableLogPrerequisite(boolean enablelog, String statusdir) throws BadParam {
if (enablelog && !TempletonUtils.isset(statusdir))
throw new BadParam("enablelog is only applicable when statusdir is set");
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java Tue Sep 2 16:30:33 2014
@@ -169,9 +169,9 @@ public class JobState {
String childJobIDs = getField("children");
if (childJobIDs != null) {
for (String jobid : childJobIDs.split(",")) {
- children.add(new JobState(jobid, config));
+ children.add(new JobState(jobid, config));
}
- }
+ }
return children;
}
Modified: hive/trunk/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hive/trunk/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java (original)
+++ hive/trunk/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java Tue Sep 2 16:30:33 2014
@@ -160,7 +160,7 @@ public abstract class HiveBaseResultSet
}
public InputStream getBinaryStream(String columnName) throws SQLException {
- return getBinaryStream(findColumn(columnName));
+ return getBinaryStream(findColumn(columnName));
}
public Blob getBlob(int i) throws SQLException {
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java Tue Sep 2 16:30:33 2014
@@ -65,20 +65,20 @@ public class StatObjectConverter {
if (statsObj.getStatsData().isSetBooleanStats()) {
BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
mColStats.setBooleanStats(
- boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null,
+ boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null,
boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null,
boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
} else if (statsObj.getStatsData().isSetLongStats()) {
LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
mColStats.setLongStats(
- longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
+ longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
longStats.isSetNumDVs() ? longStats.getNumDVs() : null,
longStats.isSetLowValue() ? longStats.getLowValue() : null,
longStats.isSetHighValue() ? longStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDoubleStats()) {
DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
mColStats.setDoubleStats(
- doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
+ doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
@@ -87,20 +87,20 @@ public class StatObjectConverter {
String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null;
String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null;
mColStats.setDecimalStats(
- decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
- decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
+ decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
+ decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
low, high);
} else if (statsObj.getStatsData().isSetStringStats()) {
StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
mColStats.setStringStats(
- stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
+ stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null,
- stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
+ stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
} else if (statsObj.getStatsData().isSetBinaryStats()) {
BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
mColStats.setBinaryStats(
- binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
+ binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null,
binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
}
@@ -109,9 +109,9 @@ public class StatObjectConverter {
public static void setFieldsIntoOldStats(
MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) {
- if (mStatsObj.getAvgColLen() != null) {
- oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
- }
+ if (mStatsObj.getAvgColLen() != null) {
+ oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
+ }
if (mStatsObj.getLongHighValue() != null) {
oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
}
@@ -131,19 +131,19 @@ public class StatObjectConverter {
oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue());
}
if (mStatsObj.getMaxColLen() != null) {
- oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
+ oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
}
if (mStatsObj.getNumDVs() != null) {
- oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
+ oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
}
if (mStatsObj.getNumFalses() != null) {
- oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
+ oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
}
if (mStatsObj.getNumTrues() != null) {
- oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
+ oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
}
if (mStatsObj.getNumNulls() != null) {
- oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
+ oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
}
oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
}
@@ -152,13 +152,13 @@ public class StatObjectConverter {
MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStatsObj) {
if (mStatsObj.getAvgColLen() != null) {
oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
- }
+ }
if (mStatsObj.getLongHighValue() != null) {
- oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
- }
- if (mStatsObj.getDoubleHighValue() != null) {
- oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
- }
+ oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
+ }
+ if (mStatsObj.getDoubleHighValue() != null) {
+ oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
+ }
oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
if (mStatsObj.getLongLowValue() != null) {
oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue());
@@ -292,20 +292,20 @@ public class StatObjectConverter {
if (statsObj.getStatsData().isSetBooleanStats()) {
BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
mColStats.setBooleanStats(
- boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null,
+ boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null,
boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null,
boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
} else if (statsObj.getStatsData().isSetLongStats()) {
LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
mColStats.setLongStats(
- longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
+ longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
longStats.isSetNumDVs() ? longStats.getNumDVs() : null,
longStats.isSetLowValue() ? longStats.getLowValue() : null,
longStats.isSetHighValue() ? longStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDoubleStats()) {
DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
mColStats.setDoubleStats(
- doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
+ doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
@@ -314,20 +314,20 @@ public class StatObjectConverter {
String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null;
String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null;
mColStats.setDecimalStats(
- decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
- decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
+ decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
+ decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
low, high);
} else if (statsObj.getStatsData().isSetStringStats()) {
StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
mColStats.setStringStats(
- stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
+ stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null,
- stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
+ stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
} else if (statsObj.getStatsData().isSetBinaryStats()) {
BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
mColStats.setBinaryStats(
- binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
+ binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null,
binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Sep 2 16:30:33 2014
@@ -348,17 +348,17 @@ public class MoveTask extends Task<MoveW
// want to isolate any potential issue it may introduce.
ArrayList<LinkedHashMap<String, String>> dp =
db.loadDynamicPartitions(
- tbd.getSourcePath(),
- tbd.getTable().getTableName(),
- tbd.getPartitionSpec(),
- tbd.getReplace(),
- dpCtx.getNumDPCols(),
- tbd.getHoldDDLTime(),
- isSkewedStoredAsDirs(tbd));
+ tbd.getSourcePath(),
+ tbd.getTable().getTableName(),
+ tbd.getPartitionSpec(),
+ tbd.getReplace(),
+ dpCtx.getNumDPCols(),
+ tbd.getHoldDDLTime(),
+ isSkewedStoredAsDirs(tbd));
if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
throw new HiveException("This query creates no partitions." +
- " To turn off this error, set hive.error.on.empty.partition=false.");
+ " To turn off this error, set hive.error.on.empty.partition=false.");
}
// for each partition spec, get the partition
@@ -412,13 +412,13 @@ public class MoveTask extends Task<MoveW
numBuckets, sortCols);
}
- dc = new DataContainer(table.getTTable(), partn.getTPartition());
- // add this partition to post-execution hook
- if (work.getOutputs() != null) {
- work.getOutputs().add(new WriteEntity(partn,
+ dc = new DataContainer(table.getTTable(), partn.getTPartition());
+ // add this partition to post-execution hook
+ if (work.getOutputs() != null) {
+ work.getOutputs().add(new WriteEntity(partn,
(tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE
: WriteEntity.WriteType.INSERT)));
- }
+ }
}
}
if (SessionState.get() != null && dc != null) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Tue Sep 2 16:30:33 2014
@@ -45,61 +45,60 @@ import org.apache.hadoop.hive.serde2.obj
public class PTFOperator extends Operator<PTFDesc> implements Serializable {
- private static final long serialVersionUID = 1L;
- boolean isMapOperator;
+ private static final long serialVersionUID = 1L;
+ boolean isMapOperator;
- transient KeyWrapperFactory keyWrapperFactory;
- protected transient KeyWrapper currentKeys;
- protected transient KeyWrapper newKeys;
- /*
- * for map-side invocation of PTFs, we cannot utilize the currentkeys null check
- * to decide on invoking startPartition in streaming mode. Hence this extra flag.
- */
- transient boolean firstMapRow;
- transient Configuration hiveConf;
- transient PTFInvocation ptfInvocation;
-
- /*
- * 1. Find out if the operator is invoked at Map-Side or Reduce-side
- * 2. Get the deserialized QueryDef
- * 3. Reconstruct the transient variables in QueryDef
- * 4. Create input partition to store rows coming from previous operator
- */
- @Override
- protected void initializeOp(Configuration jobConf) throws HiveException {
- hiveConf = jobConf;
- // if the parent is ExtractOperator, this invocation is from reduce-side
- isMapOperator = conf.isMapSide();
-
- reconstructQueryDef(hiveConf);
-
- if (isMapOperator) {
- PartitionedTableFunctionDef tDef = conf.getStartOfChain();
- outputObjInspector = tDef.getRawInputShape().getOI();
- } else {
- outputObjInspector = conf.getFuncDef().getOutputShape().getOI();
- }
-
- setupKeysWrapper(inputObjInspectors[0]);
-
- ptfInvocation = setupChain();
- ptfInvocation.initializeStreaming(jobConf, isMapOperator);
- firstMapRow = true;
-
- super.initializeOp(jobConf);
- }
-
- @Override
- protected void closeOp(boolean abort) throws HiveException {
- super.closeOp(abort);
+ transient KeyWrapperFactory keyWrapperFactory;
+ protected transient KeyWrapper currentKeys;
+ protected transient KeyWrapper newKeys;
+ /*
+ * for map-side invocation of PTFs, we cannot utilize the currentkeys null check
+ * to decide on invoking startPartition in streaming mode. Hence this extra flag.
+ */
+ transient boolean firstMapRow;
+ transient Configuration hiveConf;
+ transient PTFInvocation ptfInvocation;
+
+ /*
+ * 1. Find out if the operator is invoked at Map-Side or Reduce-side
+ * 2. Get the deserialized QueryDef
+ * 3. Reconstruct the transient variables in QueryDef
+ * 4. Create input partition to store rows coming from previous operator
+ */
+ @Override
+ protected void initializeOp(Configuration jobConf) throws HiveException {
+ hiveConf = jobConf;
+ // if the parent is ExtractOperator, this invocation is from reduce-side
+ isMapOperator = conf.isMapSide();
+
+ reconstructQueryDef(hiveConf);
+
+ if (isMapOperator) {
+ PartitionedTableFunctionDef tDef = conf.getStartOfChain();
+ outputObjInspector = tDef.getRawInputShape().getOI();
+ } else {
+ outputObjInspector = conf.getFuncDef().getOutputShape().getOI();
+ }
+
+ setupKeysWrapper(inputObjInspectors[0]);
+
+ ptfInvocation = setupChain();
+ ptfInvocation.initializeStreaming(jobConf, isMapOperator);
+ firstMapRow = true;
+
+ super.initializeOp(jobConf);
+ }
+
+ @Override
+ protected void closeOp(boolean abort) throws HiveException {
+ super.closeOp(abort);
ptfInvocation.finishPartition();
ptfInvocation.close();
}
- @Override
- public void processOp(Object row, int tag) throws HiveException
- {
- if (!isMapOperator ) {
+ @Override
+ public void processOp(Object row, int tag) throws HiveException {
+ if (!isMapOperator ) {
/*
* checkif current row belongs to the current accumulated Partition:
* - If not:
@@ -129,51 +128,51 @@ public class PTFOperator extends Operato
}
ptfInvocation.processRow(row);
- }
+ }
+
+ /**
+ * Initialize the visitor to use the QueryDefDeserializer Use the order
+ * defined in QueryDefWalker to visit the QueryDef
+ *
+ * @param hiveConf
+ * @throws HiveException
+ */
+ protected void reconstructQueryDef(Configuration hiveConf) throws HiveException {
+
+ PTFDeserializer dS =
+ new PTFDeserializer(conf, (StructObjectInspector)inputObjInspectors[0], hiveConf);
+ dS.initializePTFChain(conf.getFuncDef());
+ }
+
+ protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException {
+ PartitionDef pDef = conf.getStartOfChain().getPartition();
+ List<PTFExpressionDef> exprs = pDef.getExpressions();
+ int numExprs = exprs.size();
+ ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs];
+ ObjectInspector[] keyOIs = new ObjectInspector[numExprs];
+ ObjectInspector[] currentKeyOIs = new ObjectInspector[numExprs];
+
+ for(int i=0; i<numExprs; i++) {
+ PTFExpressionDef exprDef = exprs.get(i);
+ /*
+ * Why cannot we just use the ExprNodeEvaluator on the column?
+ * - because on the reduce-side it is initialized based on the rowOI of the HiveTable
+ * and not the OI of the ExtractOp ( the parent of this Operator on the reduce-side)
+ */
+ keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode());
+ keyOIs[i] = keyFields[i].initialize(inputOI);
+ currentKeyOIs[i] =
+ ObjectInspectorUtils.getStandardObjectInspector(keyOIs[i],
+ ObjectInspectorCopyOption.WRITABLE);
+ }
+
+ keyWrapperFactory = new KeyWrapperFactory(keyFields, keyOIs, currentKeyOIs);
+ newKeys = keyWrapperFactory.getKeyWrapper();
+ }
- /**
- * Initialize the visitor to use the QueryDefDeserializer Use the order
- * defined in QueryDefWalker to visit the QueryDef
- *
- * @param hiveConf
- * @throws HiveException
- */
- protected void reconstructQueryDef(Configuration hiveConf) throws HiveException {
-
- PTFDeserializer dS =
- new PTFDeserializer(conf, (StructObjectInspector)inputObjInspectors[0], hiveConf);
- dS.initializePTFChain(conf.getFuncDef());
- }
-
- protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException {
- PartitionDef pDef = conf.getStartOfChain().getPartition();
- List<PTFExpressionDef> exprs = pDef.getExpressions();
- int numExprs = exprs.size();
- ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs];
- ObjectInspector[] keyOIs = new ObjectInspector[numExprs];
- ObjectInspector[] currentKeyOIs = new ObjectInspector[numExprs];
-
- for(int i=0; i<numExprs; i++) {
- PTFExpressionDef exprDef = exprs.get(i);
- /*
- * Why cannot we just use the ExprNodeEvaluator on the column?
- * - because on the reduce-side it is initialized based on the rowOI of the HiveTable
- * and not the OI of the ExtractOp ( the parent of this Operator on the reduce-side)
- */
- keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode());
- keyOIs[i] = keyFields[i].initialize(inputOI);
- currentKeyOIs[i] =
- ObjectInspectorUtils.getStandardObjectInspector(keyOIs[i],
- ObjectInspectorCopyOption.WRITABLE);
- }
-
- keyWrapperFactory = new KeyWrapperFactory(keyFields, keyOIs, currentKeyOIs);
- newKeys = keyWrapperFactory.getKeyWrapper();
- }
-
- /**
- * @return the name of the operator
- */
+ /**
+ * @return the name of the operator
+ */
@Override
public String getName() {
return getOperatorName();
@@ -184,11 +183,11 @@ public class PTFOperator extends Operato
}
- @Override
- public OperatorType getType() {
- return OperatorType.PTF;
- }
-
+ @Override
+ public OperatorType getType() {
+ return OperatorType.PTF;
+ }
+
private PTFInvocation setupChain() {
Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>();
PTFInputDef iDef = conf.getFuncDef();
@@ -197,9 +196,9 @@ public class PTFOperator extends Operato
fnDefs.push((PartitionedTableFunctionDef) iDef);
iDef = ((PartitionedTableFunctionDef) iDef).getInput();
}
-
+
PTFInvocation curr = null, first = null;
-
+
while(!fnDefs.isEmpty()) {
PartitionedTableFunctionDef currFn = fnDefs.pop();
curr = new PTFInvocation(curr, currFn.getTFunction());
@@ -222,26 +221,26 @@ public class PTFOperator extends Operato
llFn.setpItr(pItr);
}
}
-
+
/*
* Responsible for the flow of rows through the PTF Chain.
- * An Invocation wraps a TableFunction.
- * The PTFOp hands the chain each row through the processRow call.
+ * An Invocation wraps a TableFunction.
+ * The PTFOp hands the chain each row through the processRow call.
* It also notifies the chain of when a Partition starts/finishes.
- *
+ *
* There are several combinations depending
* whether the TableFunction and its successor support Streaming or Batch mode.
- *
+ *
* Combination 1: Streaming + Streaming
* - Start Partition: invoke startPartition on tabFn.
- * - Process Row: invoke process Row on tabFn.
+ * - Process Row: invoke process Row on tabFn.
* Any output rows hand to next tabFn in chain or forward to next Operator.
* - Finish Partition: invoke finishPartition on tabFn.
* Any output rows hand to next tabFn in chain or forward to next Operator.
- *
+ *
* Combination 2: Streaming + Batch
* same as Combination 1
- *
+ *
* Combination 3: Batch + Batch
* - Start Partition: create or reset the Input Partition for the tabFn
* caveat is: if prev is also batch and it is not providing an Output Iterator
@@ -251,22 +250,22 @@ public class PTFOperator extends Operato
* If function gives an Output Partition: set it on next Invocation's Input Partition
* If function gives an Output Iterator: iterate and call processRow on next Invocation.
* For last Invocation in chain: forward rows to next Operator.
- *
+ *
* Combination 3: Batch + Stream
* Similar to Combination 3, except Finish Partition behavior slightly different
* - Finish Partition : invoke evaluate on tabFn on Input Partition
* iterate output rows: hand to next tabFn in chain or forward to next Operator.
- *
+ *
*/
class PTFInvocation {
-
+
PTFInvocation prev;
PTFInvocation next;
TableFunctionEvaluator tabFn;
PTFPartition inputPart;
PTFPartition outputPart;
Iterator<Object> outputPartRowsItr;
-
+
public PTFInvocation(PTFInvocation prev, TableFunctionEvaluator tabFn) {
this.prev = prev;
this.tabFn = tabFn;
@@ -274,19 +273,19 @@ public class PTFOperator extends Operato
prev.next = this;
}
}
-
+
boolean isOutputIterator() {
return tabFn.canAcceptInputAsStream() || tabFn.canIterateOutput();
}
-
+
boolean isStreaming() {
return tabFn.canAcceptInputAsStream();
}
-
+
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException {
PartitionedTableFunctionDef tabDef = tabFn.getTableDef();
PTFInputDef inputDef = tabDef.getInput();
- ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
+ ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
inputObjInspectors[0] : inputDef.getOutputShape().getOI();
tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide);
@@ -295,7 +294,7 @@ public class PTFOperator extends Operato
next.initializeStreaming(cfg, isMapSide);
}
}
-
+
void startPartition() throws HiveException {
if ( isStreaming() ) {
tabFn.startPartition();
@@ -312,7 +311,7 @@ public class PTFOperator extends Operato
next.startPartition();
}
}
-
+
void processRow(Object row) throws HiveException {
if ( isStreaming() ) {
handleOutputRows(tabFn.processRow(row));
@@ -320,7 +319,7 @@ public class PTFOperator extends Operato
inputPart.append(row);
}
}
-
+
void handleOutputRows(List<Object> outRows) throws HiveException {
if ( outRows != null ) {
for (Object orow : outRows ) {
@@ -332,7 +331,7 @@ public class PTFOperator extends Operato
}
}
}
-
+
void finishPartition() throws HiveException {
if ( isStreaming() ) {
handleOutputRows(tabFn.finishPartition());
@@ -353,7 +352,7 @@ public class PTFOperator extends Operato
}
}
}
-
+
if ( next != null ) {
next.finishPartition();
} else {
@@ -364,7 +363,7 @@ public class PTFOperator extends Operato
}
}
}
-
+
/**
* Create a new Partition.
* A partition has 2 OIs: the OI for the rows being put in and the OI for the rows
@@ -388,7 +387,7 @@ public class PTFOperator extends Operato
private void createInputPartition() throws HiveException {
PartitionedTableFunctionDef tabDef = tabFn.getTableDef();
PTFInputDef inputDef = tabDef.getInput();
- ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
+ ObjectInspector inputOI = conf.getStartOfChain() == tabDef ?
inputObjInspectors[0] : inputDef.getOutputShape().getOI();
SerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() :
@@ -400,7 +399,7 @@ public class PTFOperator extends Operato
(StructObjectInspector) inputOI,
outputOI);
}
-
+
void close() {
if ( inputPart != null ) {
inputPart.close();
@@ -411,5 +410,5 @@ public class PTFOperator extends Operato
}
}
}
-
+
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java Tue Sep 2 16:30:33 2014
@@ -27,14 +27,13 @@ import java.lang.annotation.Target;
import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction;
@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.TYPE})
+@Target(ElementType.TYPE)
@Documented
-public @interface PartitionTableFunctionDescription
-{
- Description description ();
+public @interface PartitionTableFunctionDescription {
+ Description description ();
- /**
- * if true it is not usable in the language. {@link WindowingTableFunction} is the only internal function.
- */
- boolean isInternal() default false;
+ /**
+ * if true it is not usable in the language. {@link WindowingTableFunction} is the only internal function.
+ */
+ boolean isInternal() default false;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Sep 2 16:30:33 2014
@@ -1363,8 +1363,8 @@ public final class Utilities {
codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
}
- return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec,
- progressable));
+ return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec,
+ progressable);
}
@@ -3525,7 +3525,7 @@ public final class Utilities {
return createDirsWithPermission(conf, mkdir, fsPermission, recursive);
}
- private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask,
+ private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask,
String origUmask, FileSystem fs) throws IOException {
if (unsetUmask) {
if (origUmask != null) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java Tue Sep 2 16:30:33 2014
@@ -28,39 +28,38 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction;
@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.TYPE})
+@Target(ElementType.TYPE)
@Documented
-public @interface WindowFunctionDescription
-{
- Description description ();
- /**
- * controls whether this function can be applied to a Window.
- * <p>
- * Ranking function: Rank, Dense_Rank, Percent_Rank and Cume_Dist don't operate on Windows.
- * Why? a window specification implies a row specific range i.e. every row gets its own set of rows to process the UDAF on.
- * For ranking defining a set of rows for every row makes no sense.
- * <p>
- * All other UDAFs can be computed for a Window.
- */
- boolean supportsWindow() default true;
- /**
- * A WindowFunc is implemented as {@link GenericUDAFResolver2}. It returns only one value.
- * If this is true then the function must return a List which is taken to be the column for this function in the Output table returned by the
- * {@link WindowingTableFunction}. Otherwise the output is assumed to be a single value, the column of the Output will contain the same value
- * for all the rows.
- */
- boolean pivotResult() default false;
+public @interface WindowFunctionDescription {
+ Description description ();
+ /**
+ * controls whether this function can be applied to a Window.
+ * <p>
+ * Ranking function: Rank, Dense_Rank, Percent_Rank and Cume_Dist don't operate on Windows.
+ * Why? a window specification implies a row specific range i.e. every row gets its own set of rows to process the UDAF on.
+ * For ranking defining a set of rows for every row makes no sense.
+ * <p>
+ * All other UDAFs can be computed for a Window.
+ */
+ boolean supportsWindow() default true;
+ /**
+ * A WindowFunc is implemented as {@link GenericUDAFResolver2}. It returns only one value.
+ * If this is true then the function must return a List which is taken to be the column for this function in the Output table returned by the
+ * {@link WindowingTableFunction}. Otherwise the output is assumed to be a single value, the column of the Output will contain the same value
+ * for all the rows.
+ */
+ boolean pivotResult() default false;
- /**
- * Used in translations process to validate arguments
- * @return true if ranking function
- */
- boolean rankingFunction() default false;
+ /**
+ * Used in translations process to validate arguments
+ * @return true if ranking function
+ */
+ boolean rankingFunction() default false;
- /**
- * Using in analytical functions to specify that UDF implies an ordering
- * @return true if the function implies order
- */
- boolean impliesOrder() default false;
+ /**
+ * Using in analytical functions to specify that UDF implies an ordering
+ * @return true if the function implies order
+ */
+ boolean impliesOrder() default false;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java Tue Sep 2 16:30:33 2014
@@ -22,45 +22,39 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hive.common.util.AnnotationUtils;
@SuppressWarnings("deprecation")
-public class WindowFunctionInfo implements CommonFunctionInfo
-{
- boolean supportsWindow = true;
- boolean pivotResult = false;
- boolean impliesOrder = false;
- FunctionInfo fInfo;
-
- WindowFunctionInfo(FunctionInfo fInfo)
- {
- assert fInfo.isGenericUDAF();
- this.fInfo = fInfo;
- Class<? extends GenericUDAFResolver> wfnCls = fInfo.getGenericUDAFResolver().getClass();
- WindowFunctionDescription def =
+public class WindowFunctionInfo implements CommonFunctionInfo {
+ boolean supportsWindow = true;
+ boolean pivotResult = false;
+ boolean impliesOrder = false;
+ FunctionInfo fInfo;
+
+ WindowFunctionInfo(FunctionInfo fInfo) {
+ assert fInfo.isGenericUDAF();
+ this.fInfo = fInfo;
+ Class<? extends GenericUDAFResolver> wfnCls = fInfo.getGenericUDAFResolver().getClass();
+ WindowFunctionDescription def =
AnnotationUtils.getAnnotation(wfnCls, WindowFunctionDescription.class);
- if ( def != null)
- {
- supportsWindow = def.supportsWindow();
- pivotResult = def.pivotResult();
- impliesOrder = def.impliesOrder();
- }
- }
-
- public boolean isSupportsWindow()
- {
- return supportsWindow;
- }
-
- public boolean isPivotResult()
- {
- return pivotResult;
- }
-
- public boolean isImpliesOrder(){
- return impliesOrder;
- }
- public FunctionInfo getfInfo()
- {
- return fInfo;
- }
+ if ( def != null) {
+ supportsWindow = def.supportsWindow();
+ pivotResult = def.pivotResult();
+ impliesOrder = def.impliesOrder();
+ }
+ }
+
+ public boolean isSupportsWindow() {
+ return supportsWindow;
+ }
+
+ public boolean isPivotResult() {
+ return pivotResult;
+ }
+
+ public boolean isImpliesOrder() {
+ return impliesOrder;
+ }
+ public FunctionInfo getfInfo() {
+ return fInfo;
+ }
@Override
public Class<?> getFunctionClass() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java Tue Sep 2 16:30:33 2014
@@ -63,7 +63,7 @@ public class MapJoinMemoryExhaustionHand
if(maxHeapSize == -1) {
this.maxHeapSize = 200L * 1024L * 1024L;
LOG.warn("MemoryMXBean.getHeapMemoryUsage().getMax() returned -1, " +
- "defaulting maxHeapSize to 200MB");
+ "defaulting maxHeapSize to 200MB");
} else {
this.maxHeapSize = maxHeapSize;
}
@@ -91,4 +91,4 @@ public class MapJoinMemoryExhaustionHand
throw new MapJoinMemoryExhaustionException(msg);
}
}
-}
\ No newline at end of file
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java Tue Sep 2 16:30:33 2014
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.Writable;
@SuppressWarnings("deprecation")
public class MapJoinTableContainerSerDe {
-
+
private final MapJoinObjectSerDeContext keyContext;
private final MapJoinObjectSerDeContext valueContext;
public MapJoinTableContainerSerDe(MapJoinObjectSerDeContext keyContext,
@@ -70,7 +70,7 @@ public class MapJoinTableContainerSerDe
}
try {
Writable keyContainer = keySerDe.getSerializedClass().newInstance();
- Writable valueContainer = valueSerDe.getSerializedClass().newInstance();
+ Writable valueContainer = valueSerDe.getSerializedClass().newInstance();
int numKeys = in.readInt();
for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) {
MapJoinKeyObject key = new MapJoinKeyObject();
@@ -89,7 +89,7 @@ public class MapJoinTableContainerSerDe
public void persist(ObjectOutputStream out, MapJoinPersistableTableContainer tableContainer)
throws HiveException {
int numKeys = tableContainer.size();
- try {
+ try {
out.writeUTF(tableContainer.getClass().getName());
out.writeObject(tableContainer.getMetaData());
out.writeInt(numKeys);
@@ -108,7 +108,7 @@ public class MapJoinTableContainerSerDe
throw new ConcurrentModificationException("TableContainer was modified while persisting: " + tableContainer);
}
}
-
+
public static void persistDummyTable(ObjectOutputStream out) throws IOException {
MapJoinPersistableTableContainer tableContainer = new HashMapWrapper();
out.writeUTF(tableContainer.getClass().getName());
@@ -127,8 +127,8 @@ public class MapJoinTableContainerSerDe
return constructor.newInstance(metaData);
} catch (Exception e) {
String msg = "Error while attemping to create table container" +
- " of type: " + name + ", with metaData: " + metaData;
+ " of type: " + name + ", with metaData: " + metaData;
throw new HiveException(msg, e);
}
}
-}
\ No newline at end of file
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java Tue Sep 2 16:30:33 2014
@@ -270,7 +270,7 @@ public class PTFRowContainer<Row extends
FileSystem fs = finalOutPath.getFileSystem(jc);
final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
- BytesWritable.class, valueClass, isCompressed, progress);
+ BytesWritable.class, valueClass, isCompressed, progress);
return new PTFRecordWriter(outStream);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java Tue Sep 2 16:30:33 2014
@@ -146,7 +146,7 @@ public class VectorHashKeyWrapper extend
duplicateTo(clone);
return clone;
}
-
+
public void duplicateTo(VectorHashKeyWrapper clone) {
clone.longValues = longValues.clone();
clone.doubleValues = doubleValues.clone();
@@ -155,7 +155,7 @@ public class VectorHashKeyWrapper extend
// Decimal128 requires deep clone
clone.decimalValues = new Decimal128[decimalValues.length];
for(int i = 0; i < decimalValues.length; ++i) {
- clone.decimalValues[i] = new Decimal128().update(decimalValues[i]);
+ clone.decimalValues[i] = new Decimal128().update(decimalValues[i]);
}
clone.byteValues = new byte[byteValues.length][];
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Tue Sep 2 16:30:33 2014
@@ -325,11 +325,11 @@ public class VectorizationContext {
childExpressions, mode, exprDesc.getTypeInfo());
}
} else if (exprDesc instanceof ExprNodeNullDesc) {
- ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
+ ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
} else if (exprDesc instanceof ExprNodeConstantDesc) {
ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),
mode);
- }
+ }
if (ve == null) {
throw new HiveException("Could not vectorize expression: "+exprDesc.getName());
}
@@ -413,8 +413,8 @@ public class VectorizationContext {
}
}
} else {
- for (ExprNodeDesc child : children) {
- ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
+ for (ExprNodeDesc child : children) {
+ ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
if (castExpression != null) {
atleastOneCastNeeded = true;
childrenWithCasts.add(castExpression);
@@ -515,7 +515,7 @@ public class VectorizationContext {
}
return null;
}
-
+
private int getPrecisionForType(PrimitiveTypeInfo typeInfo) {
if (isFloatFamily(typeInfo.getTypeName())) {
return HiveDecimal.MAX_PRECISION;
@@ -572,8 +572,8 @@ public class VectorizationContext {
((GenericUDFBridge) genericUdf).setUdfClassName(udfClass.getClass().getName());
}
if (genericUdf instanceof SettableUDF) {
- ((SettableUDF)genericUdf).setTypeInfo(castType);
- }
+ ((SettableUDF) genericUdf).setTypeInfo(castType);
+ }
return genericUdf;
}
@@ -662,63 +662,63 @@ public class VectorizationContext {
* @throws HiveException
*/
ExprNodeDesc evaluateCastOnConstants(ExprNodeDesc exprDesc) throws HiveException {
- if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
- return exprDesc;
- }
-
- if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
- return exprDesc;
- }
-
- ExprNodeConstantDesc foldedChild = null;
- if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
-
- // try recursive folding
- ExprNodeDesc expr = evaluateCastOnConstants(exprDesc.getChildren().get(0));
- if (expr instanceof ExprNodeConstantDesc) {
- foldedChild = (ExprNodeConstantDesc) expr;
- }
- } else {
- foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
- }
-
- if (foldedChild == null) {
- return exprDesc;
- }
-
- ObjectInspector childoi = foldedChild.getWritableObjectInspector();
- GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
-
- // Only evaluate +ve/-ve or cast on constant or recursive casting.
- if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive ||
- castExpressionUdfs.contains(gudf.getClass())
- || ((gudf instanceof GenericUDFBridge)
- && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
- ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
- ObjectInspector output = evaluator.initialize(childoi);
- Object constant = evaluator.evaluate(null);
- Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
- return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
- }
-
- return exprDesc;
+ if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
+ return exprDesc;
+ }
+
+ if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
+ return exprDesc;
+ }
+
+ ExprNodeConstantDesc foldedChild = null;
+ if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
+
+ // try recursive folding
+ ExprNodeDesc expr = evaluateCastOnConstants(exprDesc.getChildren().get(0));
+ if (expr instanceof ExprNodeConstantDesc) {
+ foldedChild = (ExprNodeConstantDesc) expr;
+ }
+ } else {
+ foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
+ }
+
+ if (foldedChild == null) {
+ return exprDesc;
+ }
+
+ ObjectInspector childoi = foldedChild.getWritableObjectInspector();
+ GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
+
+ // Only evaluate +ve/-ve or cast on constant or recursive casting.
+ if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive ||
+ castExpressionUdfs.contains(gudf.getClass())
+ || ((gudf instanceof GenericUDFBridge)
+ && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
+ ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
+ ObjectInspector output = evaluator.initialize(childoi);
+ Object constant = evaluator.evaluate(null);
+ Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
+ return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
+ }
+
+ return exprDesc;
}
-
+
/* For cast on constant operator in all members of the input list and return new list
* containing results.
*/
private List<ExprNodeDesc> evaluateCastOnConstants(List<ExprNodeDesc> childExpr)
- throws HiveException {
- List<ExprNodeDesc> evaluatedChildren = new ArrayList<ExprNodeDesc>();
- if (childExpr != null) {
+ throws HiveException {
+ List<ExprNodeDesc> evaluatedChildren = new ArrayList<ExprNodeDesc>();
+ if (childExpr != null) {
for (ExprNodeDesc expr : childExpr) {
- expr = this.evaluateCastOnConstants(expr);
- evaluatedChildren.add(expr);
+ expr = this.evaluateCastOnConstants(expr);
+ evaluatedChildren.add(expr);
}
- }
- return evaluatedChildren;
+ }
+ return evaluatedChildren;
}
-
+
private VectorExpression getConstantVectorExpression(Object constantValue, TypeInfo typeInfo,
Mode mode) throws HiveException {
String type = typeInfo.getTypeName();
@@ -728,7 +728,7 @@ public class VectorizationContext {
outCol = ocm.allocateOutputColumn(colVectorType);
}
if (constantValue == null) {
- return new ConstantVectorExpression(outCol, type, true);
+ return new ConstantVectorExpression(outCol, type, true);
} else if (decimalTypePattern.matcher(type).matches()) {
VectorExpression ve = new ConstantVectorExpression(outCol, (Decimal128) constantValue);
ve.setOutputType(typeInfo.getTypeName());
@@ -907,9 +907,9 @@ public class VectorizationContext {
private VectorExpression getGenericUdfVectorExpression(GenericUDF udf,
List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
- List<ExprNodeDesc> castedChildren = evaluateCastOnConstants(childExpr);
- childExpr = castedChildren;
-
+ List<ExprNodeDesc> castedChildren = evaluateCastOnConstants(childExpr);
+ childExpr = castedChildren;
+
//First handle special cases
if (udf instanceof GenericUDFBetween) {
return getBetweenFilterExpression(childExpr, mode, returnType);
@@ -933,8 +933,8 @@ public class VectorizationContext {
}
} else if (udf instanceof GenericUDFToDecimal) {
return getCastToDecimal(childExpr, returnType);
- }
-
+ }
+
// Now do a general lookup
Class<?> udfClass = udf.getClass();
if (udf instanceof GenericUDFBridge) {
@@ -1003,7 +1003,7 @@ public class VectorizationContext {
}
}
}
-
+
/**
* Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
@@ -1014,8 +1014,8 @@ public class VectorizationContext {
String colType = colExpr.getTypeString();
// prepare arguments for createVectorExpression
- List<ExprNodeDesc> childrenForInList = evaluateCastOnConstants(childExpr.subList(1, childExpr.size()));
-
+ List<ExprNodeDesc> childrenForInList = evaluateCastOnConstants(childExpr.subList(1, childExpr.size()));
+
/* This method assumes that the IN list has no NULL entries. That is enforced elsewhere,
* in the Vectorizer class. If NULL is passed in as a list entry, behavior is not defined.
* If in the future, NULL values are allowed in the IN list, be sure to handle 3-valued
@@ -1110,105 +1110,105 @@ public class VectorizationContext {
return getCastToString(childExpr, returnType);
}
return null;
- }
-
+ }
+
private Decimal128 castConstantToDecimal(Object scalar, TypeInfo type) throws HiveException {
- PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
- String typename = type.getTypeName();
- Decimal128 d = new Decimal128();
- int scale = HiveDecimalUtils.getScaleForType(ptinfo);
- switch (ptinfo.getPrimitiveCategory()) {
- case FLOAT:
- float floatVal = ((Float) scalar).floatValue();
- d.update(floatVal, (short) scale);
- break;
- case DOUBLE:
- double doubleVal = ((Double) scalar).doubleValue();
- d.update(doubleVal, (short) scale);
- break;
- case BYTE:
- byte byteVal = ((Byte) scalar).byteValue();
- d.update(byteVal, (short) scale);
- break;
- case SHORT:
- short shortVal = ((Short) scalar).shortValue();
- d.update(shortVal, (short) scale);
- break;
- case INT:
- int intVal = ((Integer) scalar).intValue();
- d.update(intVal, (short) scale);
- break;
- case LONG:
- long longVal = ((Long) scalar).longValue();
- d.update(longVal, (short) scale);
- break;
- case DECIMAL:
- HiveDecimal decimalVal = (HiveDecimal) scalar;
- d.update(decimalVal.unscaledValue(), (short) scale);
- break;
- default:
- throw new HiveException("Unsupported type "+typename+" for cast to Decimal128");
- }
- return d;
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ Decimal128 d = new Decimal128();
+ int scale = HiveDecimalUtils.getScaleForType(ptinfo);
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ float floatVal = ((Float) scalar).floatValue();
+ d.update(floatVal, (short) scale);
+ break;
+ case DOUBLE:
+ double doubleVal = ((Double) scalar).doubleValue();
+ d.update(doubleVal, (short) scale);
+ break;
+ case BYTE:
+ byte byteVal = ((Byte) scalar).byteValue();
+ d.update(byteVal, (short) scale);
+ break;
+ case SHORT:
+ short shortVal = ((Short) scalar).shortValue();
+ d.update(shortVal, (short) scale);
+ break;
+ case INT:
+ int intVal = ((Integer) scalar).intValue();
+ d.update(intVal, (short) scale);
+ break;
+ case LONG:
+ long longVal = ((Long) scalar).longValue();
+ d.update(longVal, (short) scale);
+ break;
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ d.update(decimalVal.unscaledValue(), (short) scale);
+ break;
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Decimal128");
+ }
+ return d;
}
private String castConstantToString(Object scalar, TypeInfo type) throws HiveException {
- PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
- String typename = type.getTypeName();
- switch (ptinfo.getPrimitiveCategory()) {
- case FLOAT:
- case DOUBLE:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- return ((Number) scalar).toString();
- case DECIMAL:
- HiveDecimal decimalVal = (HiveDecimal) scalar;
- return decimalVal.toString();
- default:
- throw new HiveException("Unsupported type "+typename+" for cast to String");
- }
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).toString();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.toString();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to String");
+ }
}
private Double castConstantToDouble(Object scalar, TypeInfo type) throws HiveException {
- PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
- String typename = type.getTypeName();
- switch (ptinfo.getPrimitiveCategory()) {
- case FLOAT:
- case DOUBLE:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- return ((Number) scalar).doubleValue();
- case DECIMAL:
- HiveDecimal decimalVal = (HiveDecimal) scalar;
- return decimalVal.doubleValue();
- default:
- throw new HiveException("Unsupported type "+typename+" for cast to Double");
- }
- }
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).doubleValue();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.doubleValue();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Double");
+ }
+ }
private Long castConstantToLong(Object scalar, TypeInfo type) throws HiveException {
- PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
- String typename = type.getTypeName();
- switch (ptinfo.getPrimitiveCategory()) {
- case FLOAT:
- case DOUBLE:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- return ((Number) scalar).longValue();
- case DECIMAL:
- HiveDecimal decimalVal = (HiveDecimal) scalar;
- return decimalVal.longValue();
- default:
- throw new HiveException("Unsupported type "+typename+" for cast to Long");
- }
- }
-
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).longValue();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.longValue();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Long");
+ }
+ }
+
private VectorExpression getCastToDecimal(List<ExprNodeDesc> childExpr, TypeInfo returnType)
throws HiveException {
ExprNodeDesc child = childExpr.get(0);
@@ -1217,9 +1217,9 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Decimal128 decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
} else if (child instanceof ExprNodeNullDesc) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);
@@ -1234,8 +1234,8 @@ public class VectorizationContext {
return createVectorExpression(CastTimestampToDecimal.class, childExpr, Mode.PROJECTION, returnType);
}
throw new HiveException("Unhandled cast input type: " + inputType);
- }
-
+ }
+
private VectorExpression getCastToString(List<ExprNodeDesc> childExpr, TypeInfo returnType)
throws HiveException {
ExprNodeDesc child = childExpr.get(0);
@@ -1244,9 +1244,9 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
String strValue = castConstantToString(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
} else if (child instanceof ExprNodeNullDesc) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
@@ -1273,9 +1273,9 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
} else if (child instanceof ExprNodeNullDesc) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
@@ -1286,7 +1286,7 @@ public class VectorizationContext {
// float types require no conversion, so use a no-op
return getIdentityExpression(childExpr);
- }
+ }
// The string type is deliberately omitted -- it's handled elsewhere. See isLegacyPathUDF.
return null;
@@ -1301,7 +1301,7 @@ public class VectorizationContext {
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
return null;
} else if (child instanceof ExprNodeNullDesc) {
- return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
+ return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
}
// Long and double are handled using descriptors, string needs to be specially handled.
if (inputType.equals("string")) {
@@ -1329,9 +1329,9 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
+ return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
} else if (child instanceof ExprNodeNullDesc) {
- return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
+ return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
}
// Float family, timestamp are handled via descriptor based lookup, int family needs
// special handling.
@@ -1519,11 +1519,11 @@ public class VectorizationContext {
public static boolean isTimestampFamily(String resultType) {
return resultType.equalsIgnoreCase("timestamp");
}
-
+
public static boolean isDateFamily(String resultType) {
return resultType.equalsIgnoreCase("date");
}
-
+
// return true if this is any kind of float
public static boolean isFloatFamily(String resultType) {
return resultType.equalsIgnoreCase("double")
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java Tue Sep 2 16:30:33 2014
@@ -75,23 +75,23 @@ public class ConstantVectorExpression ex
this(outputColumn, "decimal");
setDecimalValue(value);
}
-
+
/*
* Support for null constant object
*/
public ConstantVectorExpression(int outputColumn, String typeString, boolean isNull) {
- this(outputColumn, typeString);
- isNullValue = isNull;
+ this(outputColumn, typeString);
+ isNullValue = isNull;
}
-
+
private void evaluateLong(VectorizedRowBatch vrg) {
LongColumnVector cv = (LongColumnVector) vrg.cols[outputColumn];
cv.isRepeating = true;
cv.noNulls = !isNullValue;
if (!isNullValue) {
- cv.vector[0] = longValue;
+ cv.vector[0] = longValue;
} else {
- cv.isNull[0] = true;
+ cv.isNull[0] = true;
}
}
@@ -100,10 +100,10 @@ public class ConstantVectorExpression ex
cv.isRepeating = true;
cv.noNulls = !isNullValue;
if (!isNullValue) {
- cv.vector[0] = doubleValue;
+ cv.vector[0] = doubleValue;
} else {
- cv.isNull[0] = true;
- }
+ cv.isNull[0] = true;
+ }
}
private void evaluateBytes(VectorizedRowBatch vrg) {
@@ -112,9 +112,9 @@ public class ConstantVectorExpression ex
cv.noNulls = !isNullValue;
cv.initBuffer();
if (!isNullValue) {
- cv.setVal(0, bytesValue, 0, bytesValueLength);
+ cv.setVal(0, bytesValue, 0, bytesValueLength);
} else {
- cv.isNull[0] = true;
+ cv.isNull[0] = true;
}
}
@@ -123,9 +123,9 @@ public class ConstantVectorExpression ex
dcv.isRepeating = true;
dcv.noNulls = !isNullValue;
if (!isNullValue) {
- dcv.vector[0].update(decimalValue);
+ dcv.vector[0].update(decimalValue);
} else {
- dcv.isNull[0] = true;
+ dcv.isNull[0] = true;
}
}
@@ -194,7 +194,7 @@ public class ConstantVectorExpression ex
} else if (VectorizationContext.isDecimalFamily(typeString)){
this.type = Type.DECIMAL;
} else {
- // everything else that does not belong to string, double, decimal is treated as long.
+ // everything else that does not belong to string, double, decimal is treated as long.
this.type = Type.LONG;
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IdentityExpression.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IdentityExpression.java?rev=1622060&r1=1622059&r2=1622060&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IdentityExpression.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IdentityExpression.java Tue Sep 2 16:30:33 2014
@@ -33,17 +33,17 @@ public class IdentityExpression extends
public IdentityExpression() {
}
- public IdentityExpression(int colNum, String type) {
- this.colNum = colNum;
+ public IdentityExpression(int colNum, String type) {
+ this.colNum = colNum;
this.type = type;
- }
+ }
- @Override
+ @Override
public void evaluate(VectorizedRowBatch batch) {
if (childExpressions != null) {
this.evaluateChildren(batch);
}
- }
+ }
@Override
public int getOutputColumn() {