You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/25 00:38:35 UTC
svn commit: r1526035 - in /hive/branches/vectorization: ./
hcatalog/src/docs/src/documentation/content/xdocs/
hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/
hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/t...
Author: hashutosh
Date: Tue Sep 24 22:38:33 2013
New Revision: 1526035
URL: http://svn.apache.org/r1526035
Log:
Merged in with latest trunk
Added:
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HiveJobIDParser.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HiveJobIDParser.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JarJobIDParser.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JarJobIDParser.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobIDParser.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobIDParser.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/PigJobIDParser.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/PigJobIDParser.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/hive/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/hive/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/hive/stderr
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/hive/stderr
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/jar/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/jar/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/jar/stderr
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/jar/stderr
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/pig/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/pig/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/pig/stderr
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/pig/stderr
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/streaming/
- copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/streaming/
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/streaming/stderr
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/streaming/stderr
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java
- copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java
- copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe4.java
- copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe4.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe5.java
- copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe5.java
hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_wise_fileformat18.q
- copied unchanged from r1526034, hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat18.q
hive/branches/vectorization/ql/src/test/results/clientpositive/partition_wise_fileformat18.q.out
- copied unchanged from r1526034, hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat18.q.out
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
- copied unchanged from r1526034, hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
Modified:
hive/branches/vectorization/ (props changed)
hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml
hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml
hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml
hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
Propchange: hive/branches/vectorization/
------------------------------------------------------------------------------
Merged /hive/trunk:r1525675-1526034
Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml Tue Sep 24 22:38:33 2013
@@ -66,6 +66,13 @@
<td>None</td>
</tr>
+ <tr><td><strong>enablelog</strong></td>
+ <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+ statusdir must be set as well to use this feature.</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
<tr><td><strong>callback</strong></td>
<td>Define a URL to be called upon job completion. You may embed a specific
job ID into this URL using <code>$jobId</code>. This tag
Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml Tue Sep 24 22:38:33 2013
@@ -86,6 +86,13 @@
<td>None</td>
</tr>
+ <tr><td><strong>enablelog</strong></td>
+ <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+ statusdir must be set as well to use this feature.</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
<tr><td><strong>callback</strong></td>
<td>Define a URL to be called upon job completion. You may embed a specific
job ID into this URL using <code>$jobId</code>. This tag
Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml Tue Sep 24 22:38:33 2013
@@ -101,6 +101,13 @@
<td>None</td>
</tr>
+ <tr><td><strong>enablelog</strong></td>
+ <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+ statusdir must be set as well to use this feature.</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
<tr><td><strong>callback</strong></td>
<td>Define a URL to be called upon job completion. You may embed a specific
job ID into this URL using <code>$jobId</code>. This tag
Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml Tue Sep 24 22:38:33 2013
@@ -70,6 +70,13 @@
<td>None</td>
</tr>
+ <tr><td><strong>enablelog</strong></td>
+ <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+ statusdir must be set as well to use this feature.</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
<tr><td><strong>callback</strong></td>
<td>Define a URL to be called upon job completion. You may embed a specific
job ID into this URL using <code>$jobId</code>. This tag
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,25 +43,25 @@ public class HiveDelegator extends Launc
public EnqueueBean run(String user,
String execute, String srcFile, List<String> defines,
List<String> hiveArgs, String otherFiles,
- String statusdir, String callback, String completedUrl)
+ String statusdir, String callback, String completedUrl, boolean enablelog)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException
{
runAs = user;
List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir,
- completedUrl);
+ completedUrl, enablelog);
return enqueueController(user, callback, args);
}
private List<String> makeArgs(String execute, String srcFile,
List<String> defines, List<String> hiveArgs, String otherFiles,
- String statusdir, String completedUrl)
+ String statusdir, String completedUrl, boolean enablelog)
throws BadParam, IOException, InterruptedException
{
ArrayList<String> args = new ArrayList<String>();
try {
- args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl));
+ args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
args.add("--");
args.add(appConf.hivePath());
@@ -99,7 +99,8 @@ public class HiveDelegator extends Launc
}
private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles,
- String statusdir, String completedUrl)
+ String statusdir, String completedUrl,
+ boolean enablelog)
throws URISyntaxException, FileNotFoundException, IOException,
InterruptedException
{
@@ -115,7 +116,8 @@ public class HiveDelegator extends Launc
allFiles.addAll(Arrays.asList(ofs));
}
- args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
+ args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
+ enablelog, JobType.HIVE));
args.add("-archives");
args.add(appConf.hiveArchive());
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Sep 24 22:38:33 2013
@@ -41,13 +41,14 @@ public class JarDelegator extends Launch
public EnqueueBean run(String user, String jar, String mainClass,
String libjars, String files,
List<String> jarArgs, List<String> defines,
- String statusdir, String callback, String completedUrl)
+ String statusdir, String callback, String completedUrl,
+ boolean enablelog, JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(jar, mainClass,
libjars, files, jarArgs, defines,
- statusdir, completedUrl);
+ statusdir, completedUrl, enablelog, jobType);
return enqueueController(user, callback, args);
}
@@ -55,7 +56,8 @@ public class JarDelegator extends Launch
private List<String> makeArgs(String jar, String mainClass,
String libjars, String files,
List<String> jarArgs, List<String> defines,
- String statusdir, String completedUrl)
+ String statusdir, String completedUrl,
+ boolean enablelog, JobType jobType)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
try {
@@ -63,7 +65,7 @@ public class JarDelegator extends Launch
allFiles.add(TempletonUtils.hadoopFsFilename(jar, appConf, runAs));
args.addAll(makeLauncherArgs(appConf, statusdir,
- completedUrl, allFiles));
+ completedUrl, allFiles, enablelog, jobType));
args.add("--");
args.add(appConf.clusterHadoop());
args.add("jar");
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,6 +43,7 @@ import org.apache.hive.hcatalog.templeto
public class LauncherDelegator extends TempletonDelegator {
private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
protected String runAs = null;
+ static public enum JobType {JAR, STREAMING, PIG, HIVE};
public LauncherDelegator(AppConfig appConf) {
super(appConf);
@@ -105,7 +106,9 @@ public class LauncherDelegator extends T
public List<String> makeLauncherArgs(AppConfig appConf, String statusdir,
String completedUrl,
- List<String> copyFiles) {
+ List<String> copyFiles,
+ boolean enablelog,
+ JobType jobType) {
ArrayList<String> args = new ArrayList<String>();
args.add("-libjars");
@@ -123,6 +126,10 @@ public class LauncherDelegator extends T
TempletonUtils.encodeArray(copyFiles));
addDef(args, TempletonControllerJob.OVERRIDE_CLASSPATH,
makeOverrideClasspath(appConf));
+ addDef(args, TempletonControllerJob.ENABLE_LOG,
+ Boolean.toString(enablelog));
+ addDef(args, TempletonControllerJob.JOB_TYPE,
+ jobType.toString());
// Hadoop queue information
addDef(args, "mapred.job.queue.name", appConf.hadoopQueueName());
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Sep 24 22:38:33 2013
@@ -42,20 +42,20 @@ public class PigDelegator extends Launch
public EnqueueBean run(String user,
String execute, String srcFile,
List<String> pigArgs, String otherFiles,
- String statusdir, String callback, String completedUrl)
+ String statusdir, String callback, String completedUrl, boolean enablelog)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(execute,
srcFile, pigArgs,
- otherFiles, statusdir, completedUrl);
+ otherFiles, statusdir, completedUrl, enablelog);
return enqueueController(user, callback, args);
}
private List<String> makeArgs(String execute, String srcFile,
List<String> pigArgs, String otherFiles,
- String statusdir, String completedUrl)
+ String statusdir, String completedUrl, boolean enablelog)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
try {
@@ -68,7 +68,7 @@ public class PigDelegator extends Launch
allFiles.addAll(Arrays.asList(ofs));
}
- args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
+ args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
args.add("-archives");
args.add(appConf.pigArchive());
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Sep 24 22:38:33 2013
@@ -50,6 +50,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.hive.hcatalog.templeton.LauncherDelegator.JobType;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
/**
@@ -591,18 +592,21 @@ public class Server {
@FormParam("cmdenv") List<String> cmdenvs,
@FormParam("arg") List<String> args,
@FormParam("statusdir") String statusdir,
- @FormParam("callback") String callback)
+ @FormParam("callback") String callback,
+ @FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
verifyParam(inputs, "input");
verifyParam(mapper, "mapper");
verifyParam(reducer, "reducer");
+
+ checkEnableLogPrerequisite(enablelog, statusdir);
StreamingDelegator d = new StreamingDelegator(appConf);
return d.run(getDoAsUser(), inputs, output, mapper, reducer,
files, defines, cmdenvs, args,
- statusdir, callback, getCompletedUrl());
+ statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING);
}
/**
@@ -618,18 +622,21 @@ public class Server {
@FormParam("arg") List<String> args,
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
- @FormParam("callback") String callback)
+ @FormParam("callback") String callback,
+ @FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
verifyParam(jar, "jar");
verifyParam(mainClass, "class");
+
+ checkEnableLogPrerequisite(enablelog, statusdir);
JarDelegator d = new JarDelegator(appConf);
return d.run(getDoAsUser(),
jar, mainClass,
libjars, files, args, defines,
- statusdir, callback, getCompletedUrl());
+ statusdir, callback, getCompletedUrl(), enablelog, JobType.JAR);
}
/**
@@ -643,18 +650,21 @@ public class Server {
@FormParam("arg") List<String> pigArgs,
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
- @FormParam("callback") String callback)
+ @FormParam("callback") String callback,
+ @FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
if (execute == null && srcFile == null)
throw new BadParam("Either execute or file parameter required");
+
+ checkEnableLogPrerequisite(enablelog, statusdir);
PigDelegator d = new PigDelegator(appConf);
return d.run(getDoAsUser(),
execute, srcFile,
pigArgs, otherFiles,
- statusdir, callback, getCompletedUrl());
+ statusdir, callback, getCompletedUrl(), enablelog);
}
/**
@@ -670,6 +680,7 @@ public class Server {
* @param defines shortcut for command line arguments "--define"
* @param statusdir where the stderr/stdout of templeton controller job goes
* @param callback callback url when the hive job finishes
+ * @param enablelog whether to collect mapreduce log into statusdir/logs
*/
@POST
@Path("hive")
@@ -680,16 +691,19 @@ public class Server {
@FormParam("files") String otherFiles,
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
- @FormParam("callback") String callback)
+ @FormParam("callback") String callback,
+ @FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
if (execute == null && srcFile == null)
throw new BadParam("Either execute or file parameter required");
+
+ checkEnableLogPrerequisite(enablelog, statusdir);
HiveDelegator d = new HiveDelegator(appConf);
return d.run(getDoAsUser(), execute, srcFile, defines, hiveArgs, otherFiles,
- statusdir, callback, getCompletedUrl());
+ statusdir, callback, getCompletedUrl(), enablelog);
}
/**
@@ -902,6 +916,7 @@ public class Server {
return theUriInfo.getBaseUri() + VERSION
+ "/internal/complete/$jobId";
}
+
/**
* Returns canonical host name from which the request is made; used for doAs validation
*/
@@ -930,4 +945,9 @@ public class Server {
return unkHost;
}
}
+
+ private void checkEnableLogPrerequisite(boolean enablelog, String statusdir) throws BadParam {
+ if (enablelog == true && !TempletonUtils.isset(statusdir))
+ throw new BadParam("enablelog is only applicable when statusdir is set");
+ }
}
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,7 +43,9 @@ public class StreamingDelegator extends
List<String> jarArgs,
String statusdir,
String callback,
- String completedUrl)
+ String completedUrl,
+ boolean enableLog,
+ JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
List<String> args = makeArgs(inputs, output, mapper, reducer,
@@ -53,7 +55,7 @@ public class StreamingDelegator extends
return d.run(user,
appConf.streamingJar(), null,
null, null, args, defines,
- statusdir, callback, completedUrl);
+ statusdir, callback, completedUrl, enableLog, jobType);
}
private List<String> makeArgs(List<String> inputs,
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Sep 24 22:38:33 2013
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -53,6 +54,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.templeton.LauncherDelegator;
/**
* A Map Reduce job that will start another job.
@@ -69,6 +71,8 @@ import org.apache.hadoop.util.ToolRunner
public class TempletonControllerJob extends Configured implements Tool {
public static final String COPY_NAME = "templeton.copy";
public static final String STATUSDIR_NAME = "templeton.statusdir";
+ public static final String ENABLE_LOG = "templeton.enablelog";
+ public static final String JOB_TYPE = "templeton.jobtype";
public static final String JAR_ARGS_NAME = "templeton.args";
public static final String OVERRIDE_CLASSPATH = "templeton.override-classpath";
@@ -155,9 +159,17 @@ public class TempletonControllerJob exte
String statusdir = conf.get(STATUSDIR_NAME);
if (statusdir != null) {
- statusdir = TempletonUtils.addUserHomeDirectoryIfApplicable(statusdir, conf.get("user.name"), conf);
+ try {
+ statusdir = TempletonUtils.addUserHomeDirectoryIfApplicable(statusdir,
+ conf.get("user.name"));
+ } catch (URISyntaxException e) {
+ throw new IOException("Invalid status dir URI", e);
+ }
}
+ Boolean enablelog = Boolean.parseBoolean(conf.get(ENABLE_LOG));
+ LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
+
ExecutorService pool = Executors.newCachedThreadPool();
executeWatcher(pool, conf, context.getJobID(),
proc.getInputStream(), statusdir, STDOUT_FNAME);
@@ -177,6 +189,13 @@ public class TempletonControllerJob exte
state.setCompleteStatus("done");
state.close();
+ if (enablelog && TempletonUtils.isset(statusdir)) {
+ System.err.println("templeton: collecting logs for " + context.getJobID().toString()
+ + " to " + statusdir + "/logs");
+ LogRetriever logRetriever = new LogRetriever(statusdir, jobType, conf);
+ logRetriever.run();
+ }
+
if (proc.exitValue() != 0)
System.err.println("templeton: job failed with exit code "
+ proc.exitValue());
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Tue Sep 24 22:38:33 2013
@@ -33,11 +33,11 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.ws.rs.core.UriBuilder;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.templeton.UgiFactory;
@@ -213,22 +213,19 @@ public class TempletonUtils {
}
}
- public static String addUserHomeDirectoryIfApplicable(String origPathStr, String user, Configuration conf) throws IOException {
- Path path = new Path(origPathStr);
- String result = origPathStr;
-
- // shortcut for s3/asv
- // If path contains scheme, user should mean an absolute path,
- // However, path.isAbsolute tell us otherwise.
- // So we skip conversion for non-hdfs.
- if (!(path.getFileSystem(conf) instanceof DistributedFileSystem)&&
- !(path.getFileSystem(conf) instanceof LocalFileSystem)) {
- return result;
- }
- if (!path.isAbsolute()) {
- result = "/user/" + user + "/" + origPathStr;
- }
- return result;
+ public static String addUserHomeDirectoryIfApplicable(String origPathStr, String user)
+ throws IOException, URISyntaxException {
+ URI uri = new URI(origPathStr);
+
+ if (uri.getPath().isEmpty()) {
+ String newPath = "/user/" + user;
+ uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
+ } else if (!new Path(uri.getPath()).isAbsolute()) {
+ String newPath = "/user/" + user + "/" + uri.getPath();
+ uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
+ } // no work needed for absolute paths
+
+ return uri.toString();
}
public static Path hadoopFsPath(String fname, final Configuration conf, String user)
@@ -254,7 +251,7 @@ public class TempletonUtils {
}
});
- fname = addUserHomeDirectoryIfApplicable(fname, user, conf);
+ fname = addUserHomeDirectoryIfApplicable(fname, user);
URI u = new URI(fname);
Path p = new Path(u).makeQualified(defaultFs);
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Tue Sep 24 22:38:33 2013
@@ -20,6 +20,7 @@ package org.apache.hive.hcatalog.templet
import java.io.File;
import java.io.FileNotFoundException;
+import java.net.URISyntaxException;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@@ -245,4 +246,32 @@ public class TestTempletonUtils {
}
}
+ @Test
+ public void testConstructingUserHomeDirectory() throws Exception {
+ String[] sources = new String[] { "output+", "/user/hadoop/output",
+ "hdfs://container", "hdfs://container/", "hdfs://container/path",
+ "output#link", "hdfs://cointaner/output#link",
+ "hdfs://container@acc/test" };
+ String[] expectedResults = new String[] { "/user/webhcat/output+",
+ "/user/hadoop/output", "hdfs://container/user/webhcat",
+ "hdfs://container/", "hdfs://container/path",
+ "/user/webhcat/output#link", "hdfs://cointaner/output#link",
+ "hdfs://container@acc/test" };
+ for (int i = 0; i < sources.length; i++) {
+ String source = sources[i];
+ String expectedResult = expectedResults[i];
+ String result = TempletonUtils.addUserHomeDirectoryIfApplicable(source,
+ "webhcat");
+ Assert.assertEquals(result, expectedResult);
+ }
+
+ String badUri = "c:\\some\\path";
+ try {
+ TempletonUtils.addUserHomeDirectoryIfApplicable(badUri, "webhcat");
+ Assert.fail("addUserHomeDirectoryIfApplicable should fail for bad URI: "
+ + badUri);
+ } catch (URISyntaxException ex) {
+ }
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Tue Sep 24 22:38:33 2013
@@ -353,6 +353,11 @@ public class FetchOperator implements Se
}
}
+ /**
+ * A cache of Object Inspector Settable Properties.
+ */
+ private static Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
+
private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
if (currPath == null) {
getNextPath();
@@ -408,7 +413,8 @@ public class FetchOperator implements Se
ObjectInspector outputOI = ObjectInspectorConverters.getConvertedOI(
serde.getObjectInspector(),
- partitionedTableOI == null ? tblSerde.getObjectInspector() : partitionedTableOI, true);
+ partitionedTableOI == null ? tblSerde.getObjectInspector() : partitionedTableOI,
+ oiSettableProperties);
partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
serde.getObjectInspector(), outputOI);
@@ -634,7 +640,7 @@ public class FetchOperator implements Se
partSerde.initialize(job, listPart.getOverlayedProperties());
partitionedTableOI = ObjectInspectorConverters.getConvertedOI(
- partSerde.getObjectInspector(), tableOI, true);
+ partSerde.getObjectInspector(), tableOI, oiSettableProperties);
if (!partitionedTableOI.equals(tableOI)) {
break;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Sep 24 22:38:33 2013
@@ -277,6 +277,8 @@ public class MapOperator extends Operato
new HashMap<TableDesc, StructObjectInspector>();
Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
try {
+ Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
+
for (String onefile : conf.getPathToAliases().keySet()) {
PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
TableDesc tableDesc = pd.getTableDesc();
@@ -310,7 +312,7 @@ public class MapOperator extends Operato
tblRawRowObjectInspector =
(StructObjectInspector) ObjectInspectorConverters.getConvertedOI(
partRawRowObjectInspector,
- tblDeserializer.getObjectInspector(), true);
+ tblDeserializer.getObjectInspector(), oiSettableProperties);
if (identityConverterTableDesc.contains(tableDesc)) {
if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java Tue Sep 24 22:38:33 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -141,7 +142,6 @@ public final class ObjectInspectorConver
if (inputOI.equals(outputOI)) {
return new IdentityConverter();
}
- // TODO: Add support for UNION once SettableUnionObjectInspector is implemented.
switch (outputOI.getCategory()) {
case PRIMITIVE:
return getConverter((PrimitiveObjectInspector) inputOI, (PrimitiveObjectInspector) outputOI);
@@ -154,6 +154,9 @@ public final class ObjectInspectorConver
case MAP:
return new MapConverter(inputOI,
(SettableMapObjectInspector) outputOI);
+ case UNION:
+ return new UnionConverter(inputOI,
+ (SettableUnionObjectInspector) outputOI);
default:
throw new RuntimeException("Hive internal error: conversion of "
+ inputOI.getTypeName() + " to " + outputOI.getTypeName()
@@ -161,19 +164,49 @@ public final class ObjectInspectorConver
}
}
+ /*
+ * getConvertedOI with caching to store settable properties of the object
+ * inspector. Caching might help when the object inspector
+ * contains complex nested data types. Caching is not explicitly required for
+ * the returned object inspector across multiple invocations since the
+ * ObjectInspectorFactory already takes care of it.
+ */
public static ObjectInspector getConvertedOI(
+ ObjectInspector inputOI, ObjectInspector outputOI,
+ Map<ObjectInspector, Boolean> oiSettableProperties
+ ) {
+ return getConvertedOI(inputOI, outputOI, oiSettableProperties, true);
+ }
+
+ /*
+ * getConvertedOI without any caching.
+ */
+ public static ObjectInspector getConvertedOI(
+ ObjectInspector inputOI,
+ ObjectInspector outputOI
+ ) {
+ return getConvertedOI(inputOI, outputOI, null, true);
+ }
+
+ /*
+ * Utility function to convert from one object inspector type to another.
+ */
+ private static ObjectInspector getConvertedOI(
ObjectInspector inputOI,
ObjectInspector outputOI,
+ Map<ObjectInspector, Boolean> oiSettableProperties,
boolean equalsCheck) {
+ ObjectInspector retOI = outputOI.getCategory() == Category.PRIMITIVE ? inputOI : outputOI;
// If the inputOI is the same as the outputOI, just return it
- if (equalsCheck && inputOI.equals(outputOI)) {
- return outputOI;
+ // If the retOI has all fields settable, return it
+ if ((equalsCheck && inputOI.equals(outputOI)) ||
+ ObjectInspectorUtils.hasAllFieldsSettable(retOI, oiSettableProperties) == true) {
+ return retOI;
}
// Return the settable equivalent object inspector for primitive categories
// For eg: for table T containing partitions p1 and p2 (possibly different
// from the table T), return the settable inspector for T. The inspector for
// T is settable recursively i.e all the nested fields are also settable.
- // TODO: Add support for UNION once SettableUnionObjectInspector is implemented.
switch (outputOI.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector primInputOI = (PrimitiveObjectInspector) inputOI;
@@ -189,7 +222,7 @@ public final class ObjectInspectorConver
for (StructField listField : listFields) {
structFieldNames.add(listField.getFieldName());
structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(),
- listField.getFieldObjectInspector(), false));
+ listField.getFieldObjectInspector(), oiSettableProperties, false));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(
structFieldNames,
@@ -198,14 +231,25 @@ public final class ObjectInspectorConver
ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
return ObjectInspectorFactory.getStandardListObjectInspector(
getConvertedOI(listOutputOI.getListElementObjectInspector(),
- listOutputOI.getListElementObjectInspector(), false));
+ listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
case MAP:
MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
return ObjectInspectorFactory.getStandardMapObjectInspector(
getConvertedOI(mapOutputOI.getMapKeyObjectInspector(),
- mapOutputOI.getMapKeyObjectInspector(), false),
+ mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false),
getConvertedOI(mapOutputOI.getMapValueObjectInspector(),
- mapOutputOI.getMapValueObjectInspector(), false));
+ mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
+ case UNION:
+ UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
+ // create a standard settable union object inspector
+ List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
+ List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(
+ unionListFields.size());
+ for (ObjectInspector listField : unionListFields) {
+ unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties,
+ false));
+ }
+ return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
default:
throw new RuntimeException("Hive internal error: conversion of "
+ inputOI.getTypeName() + " to " + outputOI.getTypeName()
@@ -335,6 +379,67 @@ public final class ObjectInspectorConver
}
/**
+ * A converter class for Union.
+ */
+ public static class UnionConverter implements Converter {
+
+ UnionObjectInspector inputOI;
+ SettableUnionObjectInspector outputOI;
+
+ List<? extends ObjectInspector> inputFields;
+ List<? extends ObjectInspector> outputFields;
+
+ ArrayList<Converter> fieldConverters;
+
+ Object output;
+
+ public UnionConverter(ObjectInspector inputOI,
+ SettableUnionObjectInspector outputOI) {
+ if (inputOI instanceof UnionObjectInspector) {
+ this.inputOI = (UnionObjectInspector)inputOI;
+ this.outputOI = outputOI;
+ inputFields = this.inputOI.getObjectInspectors();
+ outputFields = outputOI.getObjectInspectors();
+
+ // If the output has some extra fields, set them to NULL in convert().
+ int minFields = Math.min(inputFields.size(), outputFields.size());
+ fieldConverters = new ArrayList<Converter>(minFields);
+ for (int f = 0; f < minFields; f++) {
+ fieldConverters.add(getConverter(inputFields.get(f), outputFields.get(f)));
+ }
+
+ // Create an empty output object which will be populated when convert() is invoked.
+ output = outputOI.create();
+ } else if (!(inputOI instanceof VoidObjectInspector)) {
+ throw new RuntimeException("Hive internal error: conversion of " +
+ inputOI.getTypeName() + " to " + outputOI.getTypeName() +
+ "not supported yet.");
+ }
+ }
+
+ @Override
+ public Object convert(Object input) {
+ if (input == null) {
+ return null;
+ }
+
+ int minFields = Math.min(inputFields.size(), outputFields.size());
+ // Convert the fields
+ for (int f = 0; f < minFields; f++) {
+ Object outputFieldValue = fieldConverters.get(f).convert(inputOI);
+ outputOI.addField(output, (ObjectInspector)outputFieldValue);
+ }
+
+ // set the extra fields to null
+ for (int f = minFields; f < outputFields.size(); f++) {
+ outputOI.addField(output, null);
+ }
+
+ return output;
+ }
+ }
+
+ /**
* A converter class for Map.
*/
public static class MapConverter implements Converter {
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Sep 24 22:38:33 2013
@@ -46,11 +46,25 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
@@ -1003,6 +1017,138 @@ public final class ObjectInspectorUtils
return (oi instanceof ConstantObjectInspector);
}
+ private static boolean setOISettablePropertiesMap(ObjectInspector oi,
+ Map<ObjectInspector, Boolean> oiSettableProperties, boolean value) {
+ // Cache if the client asks for it, else just return the value
+ if (!(oiSettableProperties == null)) {
+ oiSettableProperties.put(oi, value);
+ }
+ return value;
+ }
+
+ private static boolean isInstanceOfSettablePrimitiveOI(PrimitiveObjectInspector oi) {
+ switch (oi.getPrimitiveCategory()) {
+ case BOOLEAN:
+ return oi instanceof SettableBooleanObjectInspector;
+ case BYTE:
+ return oi instanceof SettableByteObjectInspector;
+ case SHORT:
+ return oi instanceof SettableShortObjectInspector;
+ case INT:
+ return oi instanceof SettableIntObjectInspector;
+ case LONG:
+ return oi instanceof SettableLongObjectInspector ;
+ case FLOAT:
+ return oi instanceof SettableFloatObjectInspector;
+ case DOUBLE:
+ return oi instanceof SettableDoubleObjectInspector;
+ case STRING:
+ return oi instanceof WritableStringObjectInspector ||
+ oi instanceof JavaStringObjectInspector;
+ case VARCHAR:
+ return oi instanceof SettableHiveVarcharObjectInspector;
+ case DATE:
+ return oi instanceof SettableDateObjectInspector;
+ case TIMESTAMP:
+ return oi instanceof SettableTimestampObjectInspector;
+ case BINARY:
+ return oi instanceof SettableBinaryObjectInspector;
+ case DECIMAL:
+ return oi instanceof SettableHiveDecimalObjectInspector;
+ default:
+ throw new RuntimeException("Hive internal error inside isAssignableFromSettablePrimitiveOI "
+ + oi.getTypeName() + " not supported yet.");
+ }
+ }
+
+ private static boolean isInstanceOfSettableOI(ObjectInspector oi)
+ {
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ return isInstanceOfSettablePrimitiveOI((PrimitiveObjectInspector)oi);
+ case STRUCT:
+ return oi instanceof SettableStructObjectInspector;
+ case LIST:
+ return oi instanceof SettableListObjectInspector;
+ case MAP:
+ return oi instanceof SettableMapObjectInspector;
+ case UNION:
+ return oi instanceof SettableUnionObjectInspector;
+ default:
+ throw new RuntimeException("Hive internal error inside isAssignableFromSettableOI : "
+ + oi.getTypeName() + " not supported yet.");
+ }
+ }
+
+ /*
+ * hasAllFieldsSettable without any caching.
+ */
+ public static Boolean hasAllFieldsSettable(ObjectInspector oi) {
+ return hasAllFieldsSettable(oi, null);
+ }
+
+ /**
+ *
+ * @param oi - Input object inspector
+ * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
+ * @return - true if : (1) oi is an instance of settable<DataType>OI.
+ * (2) All the embedded object inspectors are instances of settable<DataType>OI.
+ * If (1) or (2) is false, return false.
+ */
+ public static boolean hasAllFieldsSettable(ObjectInspector oi,
+ Map<ObjectInspector, Boolean> oiSettableProperties) {
+ // If the result is already present in the cache, return it.
+ if (!(oiSettableProperties == null) &&
+ oiSettableProperties.containsKey(oi)) {
+ return oiSettableProperties.get(oi).booleanValue();
+ }
+ // If the top-level object inspector is non-settable return false
+ if (!(isInstanceOfSettableOI(oi))) {
+ return setOISettablePropertiesMap(oi, oiSettableProperties, false);
+ }
+
+ Boolean returnValue = true;
+
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ break;
+ case STRUCT:
+ StructObjectInspector structOutputOI = (StructObjectInspector) oi;
+ List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
+ for (StructField listField : listFields) {
+ if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
+ returnValue = false;
+ break;
+ }
+ }
+ break;
+ case LIST:
+ ListObjectInspector listOutputOI = (ListObjectInspector) oi;
+ returnValue = hasAllFieldsSettable(listOutputOI.getListElementObjectInspector(),
+ oiSettableProperties);
+ break;
+ case MAP:
+ MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
+ returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) &&
+ hasAllFieldsSettable(mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
+ break;
+ case UNION:
+ UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
+ List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
+ for (ObjectInspector listField : unionListFields) {
+ if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
+ returnValue = false;
+ break;
+ }
+ }
+ break;
+ default:
+ throw new RuntimeException("Hive internal error inside hasAllFieldsSettable : "
+ + oi.getTypeName() + " not supported yet.");
+ }
+ return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
+ }
+
private ObjectInspectorUtils() {
// prevent instantiation
}
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java Tue Sep 24 22:38:33 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2.objectinspector;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -30,7 +31,7 @@ import java.util.List;
* Always use the {@link ObjectInspectorFactory} to create new ObjectInspector
* objects, instead of directly creating an instance of this class.
*/
-public class StandardUnionObjectInspector implements UnionObjectInspector {
+public class StandardUnionObjectInspector extends SettableUnionObjectInspector {
private List<ObjectInspector> ois;
protected StandardUnionObjectInspector() {
@@ -116,4 +117,17 @@ public class StandardUnionObjectInspecto
return sb.toString();
}
+ @Override
+ public Object create() {
+ ArrayList<Object> a = new ArrayList<Object>();
+ return a;
+ }
+
+ @Override
+ public Object addField(Object union, ObjectInspector oi) {
+ ArrayList<Object> a = (ArrayList<Object>) union;
+ a.add(oi);
+ return a;
+ }
+
}