You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/01/28 05:33:06 UTC
svn commit: r1655217 - in /hive/branches/llap:
ant/src/org/apache/hadoop/hive/ant/
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/ itests/qtest/
itests/util/src/main/java/org/apache/hadoop/hive/accumulo/
itests/util/src/main/java/org/apache/h...
Author: gunther
Date: Wed Jan 28 04:33:05 2015
New Revision: 1655217
URL: http://svn.apache.org/r1655217
Log:
HIVE-9461: LLAP: Enable local mode tests on tez to facilitate llap testing (Gunther Hagleitner)
Modified:
hive/branches/llap/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
hive/branches/llap/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
hive/branches/llap/itests/qtest/pom.xml
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/llap/ql/src/test/templates/TestCliDriver.vm
hive/branches/llap/ql/src/test/templates/TestCompareCliDriver.vm
hive/branches/llap/ql/src/test/templates/TestNegativeCliDriver.vm
hive/branches/llap/ql/src/test/templates/TestParseNegative.vm
hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/llap/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hive/branches/llap/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Wed Jan 28 04:33:05 2015
@@ -129,6 +129,8 @@ public class QTestGenTask extends Task {
private String logDirectory;
+ private String tezDirectory;
+
private String template;
private String className;
@@ -199,6 +201,14 @@ public class QTestGenTask extends Task {
this.template = template;
}
+ public String getTezDirectory() {
+ return tezDirectory;
+ }
+
+ public void setTezDirectory(String tezDirectory) {
+ this.tezDirectory = tezDirectory;
+ }
+
public String getTemplate() {
return template;
}
@@ -348,6 +358,7 @@ public class QTestGenTask extends Task {
File outDir = null;
File resultsDir = null;
File logDir = null;
+ File tezDir = null;
try {
@@ -416,6 +427,13 @@ public class QTestGenTask extends Task {
throw new BuildException("Log Directory " + logDir.getCanonicalPath() + " does not exist");
}
+ if (tezDirectory != null) {
+ tezDir = new File(tezDirectory);
+ if (!tezDir.exists()) {
+ tezDir.mkdirs();
+ }
+ }
+
if (resultsDirectory != null) {
resultsDir = new File(resultsDirectory);
if (!resultsDir.exists()) {
@@ -467,6 +485,7 @@ public class QTestGenTask extends Task {
ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir));
}
ctx.put("logDir", relativePath(hiveRootDir, logDir));
+ ctx.put("tezDirectory", tezDir);
ctx.put("clusterMode", clusterMode);
ctx.put("hiveConfDir", escapePath(hiveConfDir));
ctx.put("hadoopVersion", hadoopVersion);
Modified: hive/branches/llap/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java (original)
+++ hive/branches/llap/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java Wed Jan 28 04:33:05 2015
@@ -88,7 +88,7 @@ public class TestLocationQueries extends
String hadoopVer, String locationSubdir)
throws Exception
{
- super(outDir, logDir, miniMr, hadoopVer, "", "");
+ super(outDir, logDir, miniMr, hadoopVer, "", "", null);
this.locationSubdir = locationSubdir;
}
}
Modified: hive/branches/llap/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/qtest/pom.xml?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/itests/qtest/pom.xml (original)
+++ hive/branches/llap/itests/qtest/pom.xml Wed Jan 28 04:33:05 2015
@@ -426,6 +426,7 @@
logFile="${project.build.directory}/testparseneggen.log"
hadoopVersion="${active.hadoop.version}"
logDirectory="${project.build.directory}/qfile-results/negative/"
+ tezDirectory="${project.build.directory}/tez"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -442,6 +443,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver"
logFile="${project.build.directory}/testclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -459,6 +461,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver"
logFile="${project.build.directory}/testnegativeclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -475,6 +478,7 @@
className="TestCompareCliDriver"
logFile="${project.build.directory}/testcompareclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientcompare/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -492,6 +496,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestMinimrCliDriver"
logFile="${project.build.directory}/testminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -513,6 +518,7 @@
className="TestMiniTezCliDriver"
logFile="${project.build.directory}/testminitezclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -534,6 +540,25 @@
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init_for_encryption.sql"
cleanupScript="q_test_cleanup_for_encryption.sql"/>
+
+ <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
+ outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
+ templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
+ queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
+ queryFile="${qfile}"
+ includeQueryFile="${minitez.query.files},${minitez.query.files.shared}"
+ queryFileRegex="${qfile_regex}"
+ clusterMode="tezlocal"
+ runDisabled="${run_disabled}"
+ hiveConfDir="${basedir}/${hive.path.to.root}/data/conf/tez"
+ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/tez"
+ className="TestMiniTezCliDriverLocal"
+ logFile="${project.build.directory}/testminitezclidrivergen.log"
+ logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
</then>
<else>
</else>
Modified: hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java (original)
+++ hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java Wed Jan 28 04:33:05 2015
@@ -26,7 +26,7 @@ public class AccumuloQTestUtil extends Q
public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr,
AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception {
- super(outDir, logDir, miniMr, null, initScript, cleanupScript);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript, null);
setup.setupWithHiveConf(conf);
super.init();
}
Modified: hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (original)
+++ hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java Wed Jan 28 04:33:05 2015
@@ -44,7 +44,7 @@ public class HBaseQTestUtil extends QTes
String initScript, String cleanupScript)
throws Exception {
- super(outDir, logDir, miniMr, null, initScript, cleanupScript);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript, null);
setup.preTest(conf);
this.conn = setup.getConnection();
super.init();
Modified: hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Wed Jan 28 04:33:05 2015
@@ -126,6 +126,7 @@ public class QTestUtil {
private final String testFiles;
protected final String outDir;
protected final String logDir;
+ protected final String tezDir;
private final TreeMap<String, String> qMap;
private final Set<String> qSkipSet;
private final Set<String> qSortSet;
@@ -237,9 +238,11 @@ public class QTestUtil {
}
}
- public QTestUtil(String outDir, String logDir, String initScript, String cleanupScript) throws
- Exception {
- this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, cleanupScript);
+ public QTestUtil(String outDir, String logDir, String initScript,
+ String cleanupScript, String tezDirectory) throws
+ Exception {
+ this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript,
+ cleanupScript, tezDirectory);
}
public String getOutputDirectory() {
@@ -287,6 +290,10 @@ public class QTestUtil {
"/build/ql/test/data/warehouse/")).toString());
}
+ if (clusterType == MiniClusterType.tezlocal) {
+ conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, tezDir + "/jars");
+ }
+
// Windows paths should be converted after MiniMrShim.setupConfiguration()
// since setupConfiguration may overwrite configuration values.
if (Shell.WINDOWS) {
@@ -299,6 +306,7 @@ public class QTestUtil {
tez,
spark,
encrypted,
+ tezlocal,
none;
public static MiniClusterType valueForString(String type) {
@@ -310,6 +318,8 @@ public class QTestUtil {
return spark;
} else if (type.equals("encrypted")) {
return encrypted;
+ } else if (type.equals("tezlocal")) {
+ return tezlocal;
} else {
return none;
}
@@ -317,9 +327,9 @@ public class QTestUtil {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer,
- String initScript, String cleanupScript)
+ String initScript, String cleanupScript, String tezDirectory)
throws Exception {
- this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript);
+ this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript, tezDirectory);
}
private String getKeyProviderURI() {
@@ -332,12 +342,17 @@ public class QTestUtil {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer, String initScript, String cleanupScript)
+ String confDir, String hadoopVer, String initScript,
+ String cleanupScript, String tezDirectory)
throws Exception {
+
this.outDir = outDir;
this.logDir = logDir;
+ this.tezDir = tezDirectory;
+
if (confDir != null && !confDir.isEmpty()) {
- HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
+ HiveConf.setHiveSiteLocation(new URL("file://"
+ + new File(confDir).toURI().getPath() + "/hive-site.xml"));
System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
}
conf = new HiveConf(Driver.class);
@@ -349,7 +364,7 @@ public class QTestUtil {
qHashQuerySet = new HashSet<String>();
qSortNHashQuerySet = new HashSet<String>();
qJavaVersionSpecificOutput = new HashSet<String>();
- this.clusterType = clusterType;
+ QTestUtil.clusterType = clusterType;
HadoopShims shims = ShimLoader.getHadoopShims();
int numberOfDataNodes = 4;
@@ -376,7 +391,11 @@ public class QTestUtil {
String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
- mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniTezCluster(conf, 4, uriString, 1, false,
+ tezDir + "/staging");
+ } else if (clusterType == MiniClusterType.tezlocal) {
+ mr = shims.getMiniTezCluster(conf, 4, uriString, 1, true,
+ tezDir + "/staging");
} else {
mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
}
@@ -834,6 +853,15 @@ public class QTestUtil {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap();
+ if (QTestUtil.clusterType == MiniClusterType.tezlocal) {
+ conf.setBoolean("tez.local.mode", true);
+ conf.set("fs.defaultFS", "file:///");
+ conf.setBoolean("tez.runtime.optimize.local.fetch", true);
+ conf.set("tez.staging-dir", tezDir + "/staging");
+ conf.setInt("tez.am.inline.task.execution.max-tasks", 2);
+ conf.setBoolean("tez.ignore.lib.uris", true);
+ }
+
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
ss.in = System.in;
Modified: hive/branches/llap/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/templates/TestCliDriver.vm?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/templates/TestCliDriver.vm (original)
+++ hive/branches/llap/ql/src/test/templates/TestCliDriver.vm Wed Jan 28 04:33:05 2015
@@ -39,10 +39,12 @@ public class $className extends TestCase
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
String cleanupScript = "$cleanupScript";
+ String tezDirectory = "$tezDirectory";
+
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory);
// do a one time initialization
qt.cleanUp();
Modified: hive/branches/llap/ql/src/test/templates/TestCompareCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/templates/TestCompareCliDriver.vm?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/templates/TestCompareCliDriver.vm (original)
+++ hive/branches/llap/ql/src/test/templates/TestCompareCliDriver.vm Wed Jan 28 04:33:05 2015
@@ -40,10 +40,11 @@ public class $className extends TestCase
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
String cleanupScript = "$cleanupScript";
+ String tezDirectory = "$tezDirectory";
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory);
// do a one time initialization
qt.cleanUp();
@@ -107,9 +108,9 @@ public class $className extends TestCase
}
}
TestSuite suite = new TestSuite();
-
-
-
+
+
+
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
#set ($eidx = $fname.indexOf('.'))
@@ -121,7 +122,7 @@ public class $className extends TestCase
suite.addTest(new $className("testCompareCliDriver_shutdown"));
return suite;
}
-
+
private Map<String, List<String>> versionFiles = new HashMap<String, List<String>>();
static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
@@ -140,7 +141,7 @@ public class $className extends TestCase
private void runTest(String tname, String fname, String fpath) throws Exception {
final String queryDirectory = HIVE_ROOT + "$queryDir";
-
+
long startTime = System.currentTimeMillis();
try {
System.err.println("Begin query: " + fname);
Modified: hive/branches/llap/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/templates/TestNegativeCliDriver.vm?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hive/branches/llap/ql/src/test/templates/TestNegativeCliDriver.vm Wed Jan 28 04:33:05 2015
@@ -41,7 +41,7 @@ public class $className extends TestCase
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
- initScript, cleanupScript);
+ initScript, cleanupScript, null);
// do a one time initialization
qt.cleanUp();
qt.createSources();
Modified: hive/branches/llap/ql/src/test/templates/TestParseNegative.vm
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/templates/TestParseNegative.vm?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/templates/TestParseNegative.vm (original)
+++ hive/branches/llap/ql/src/test/templates/TestParseNegative.vm Wed Jan 28 04:33:05 2015
@@ -32,7 +32,7 @@ public class $className extends TestCase
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
-
+
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -42,7 +42,7 @@ public class $className extends TestCase
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
- initScript, cleanupScript);
+ initScript, cleanupScript, null);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
Modified: hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Wed Jan 28 04:33:05 2015
@@ -228,7 +228,8 @@ public class Hadoop20SShims extends Hado
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException {
+ String nameNode, int numDir, boolean local,
+ String tezDir) throws IOException {
throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion());
}
@@ -494,7 +495,7 @@ public class Hadoop20SShims extends Hado
}
public class Hadoop20SFileStatus implements HdfsFileStatus {
- private FileStatus fileStatus;
+ private final FileStatus fileStatus;
public Hadoop20SFileStatus(FileStatus fileStatus) {
this.fileStatus = fileStatus;
}
@@ -620,28 +621,33 @@ public class Hadoop20SShims extends Hado
*/
public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim {
- private KerberosName kerberosName;
+ private final KerberosName kerberosName;
public KerberosNameShim(String name) {
kerberosName = new KerberosName(name);
}
+ @Override
public String getDefaultRealm() {
return kerberosName.getDefaultRealm();
}
+ @Override
public String getServiceName() {
return kerberosName.getServiceName();
}
+ @Override
public String getHostName() {
return kerberosName.getHostName();
}
+ @Override
public String getRealm() {
return kerberosName.getRealm();
}
+ @Override
public String getShortName() throws IOException {
return kerberosName.getShortName();
}
Modified: hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Wed Jan 28 04:33:05 2015
@@ -351,7 +351,23 @@ public class Hadoop23Shims extends Hadoo
*/
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException {
+ String nameNode, int numDir, boolean local, String tezDir) throws IOException {
+ if (local) {
+ conf.setBoolean("tez.local.mode", true);
+ conf.set("fs.defaultFS", "file:///");
+ conf.setBoolean("tez.runtime.optimize.local.fetch", true);
+ conf.setInt("tez.am.inline.task.execution.max-tasks", 6);
+ conf.set("tez.staging-dir", tezDir);
+ conf.setBoolean("tez.ignore.lib.uris", true);
+ conf.setInt("tez.runtime.io.sort.mb", 10);
+ conf.setInt("tez.runtime.unordered.output.buffer.size-mb", 20);
+ conf.setFloat("tez.runtime.shuffle.fetch.buffer.percent", 0.2f);
+ conf.setFloat("tez.runtime.shuffle.memory.limit.percent", 0.2f);
+ conf.setFloat("tez.runtime.task.input.post-merge.buffer.percent", 0f);
+
+ return null;
+ }
+
return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, numDir);
}
Modified: hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1655217&r1=1655216&r2=1655217&view=diff
==============================================================================
--- hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Wed Jan 28 04:33:05 2015
@@ -93,7 +93,7 @@ public interface HadoopShims {
String nameNode, int numDir) throws IOException;
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException;
+ String nameNode, int numDir, boolean local, String tezDir) throws IOException;
/**
* Shim for MiniMrCluster