You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/03 01:49:41 UTC
svn commit: r1636225 - in /hive/trunk/shims: ./
0.23/src/main/java/org/apache/hadoop/hive/shims/ aggregator/
common/src/main/java/org/apache/hadoop/hive/shims/ scheduler/
scheduler/src/ scheduler/src/main/ scheduler/src/main/java/
scheduler/src/main/ja...
Author: brock
Date: Mon Nov 3 00:49:40 2014
New Revision: 1636225
URL: http://svn.apache.org/r1636225
Log:
HIVE-8693 - Separate out fair scheduler dependency from hadoop 0.23 shim (Prasad via Brock)
Added:
hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java
hive/trunk/shims/scheduler/
hive/trunk/shims/scheduler/pom.xml
hive/trunk/shims/scheduler/src/
hive/trunk/shims/scheduler/src/main/
hive/trunk/shims/scheduler/src/main/java/
hive/trunk/shims/scheduler/src/main/java/org/
hive/trunk/shims/scheduler/src/main/java/org/apache/
hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/
hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/
hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/
hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java
Modified:
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/aggregator/pom.xml
hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
hive/trunk/shims/pom.xml
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1636225&r1=1636224&r2=1636225&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Nov 3 00:49:40 2014
@@ -30,7 +30,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -77,10 +76,6 @@ import org.apache.hadoop.security.authen
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy;
import org.apache.tez.test.MiniTezCluster;
import com.google.common.base.Joiner;
@@ -92,7 +87,6 @@ import com.google.common.collect.Iterabl
* Implemention of shims against Hadoop 0.23.0.
*/
public class Hadoop23Shims extends HadoopShimsSecure {
- private static final String MR2_JOB_QUEUE_PROPERTY = "mapreduce.job.queuename";
HadoopShims.MiniDFSShim cluster = null;
@@ -232,39 +226,13 @@ public class Hadoop23Shims extends Hadoo
*/
@Override
public void refreshDefaultQueue(Configuration conf, String userName) throws IOException {
- String requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
if (StringUtils.isNotBlank(userName) && isFairScheduler(conf)) {
- final AtomicReference<AllocationConfiguration> allocConf = new AtomicReference<AllocationConfiguration>();
-
- AllocationFileLoaderService allocsLoader = new AllocationFileLoaderService();
- allocsLoader.init(conf);
- allocsLoader.setReloadListener(new AllocationFileLoaderService.Listener() {
- @Override
- public void onReload(AllocationConfiguration allocs) {
- allocConf.set(allocs);
- }
- });
- try {
- allocsLoader.reloadAllocations();
- } catch (Exception ex) {
- throw new IOException("Failed to load queue allocations", ex);
- }
- if (allocConf.get() == null) {
- allocConf.set(new AllocationConfiguration(conf));
- }
- QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy();
- if (queuePolicy != null) {
- requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
- if (StringUtils.isNotBlank(requestedQueue)) {
- LOG.debug("Setting queue name to " + requestedQueue + " for user " + userName);
- conf.set(MR2_JOB_QUEUE_PROPERTY, requestedQueue);
- }
- }
+ ShimLoader.getSchedulerShims().refreshDefaultQueue(conf, userName);
}
}
private boolean isFairScheduler (Configuration conf) {
- return FairScheduler.class.getName().
+ return "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler".
equalsIgnoreCase(conf.get(YarnConfiguration.RM_SCHEDULER));
}
Modified: hive/trunk/shims/aggregator/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/aggregator/pom.xml?rev=1636225&r1=1636224&r2=1636225&view=diff
==============================================================================
--- hive/trunk/shims/aggregator/pom.xml (original)
+++ hive/trunk/shims/aggregator/pom.xml Mon Nov 3 00:49:40 2014
@@ -63,5 +63,11 @@
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-scheduler</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
</project>
Added: hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java?rev=1636225&view=auto
==============================================================================
--- hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java (added)
+++ hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java Mon Nov 3 00:49:40 2014
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Shim for Fair scheduler
+ * HiveServer2 uses fair scheduler API to resolve the queue mapping for non-impersonation
+ * mode. This shim is avoid direct dependency of yarn fair scheduler on Hive.
+ */
+public interface SchedulerShim {
+ /**
+ * Reset the default fair scheduler queue mapping to end user.
+ * @param conf
+ * @param userName end user name
+ */
+ public void refreshDefaultQueue(Configuration conf, String userName)
+ throws IOException;
+}
Modified: hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java?rev=1636225&r1=1636224&r2=1636225&view=diff
==============================================================================
--- hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java (original)
+++ hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java Mon Nov 3 00:49:40 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.shims;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.util.VersionInfo;
import org.apache.log4j.AppenderSkeleton;
@@ -33,6 +34,7 @@ public abstract class ShimLoader {
private static JettyShims jettyShims;
private static AppenderSkeleton eventCounter;
private static HadoopThriftAuthBridge hadoopThriftAuthBridge;
+ private static SchedulerShim schedulerShim;
/**
* The names of the classes for shimming Hadoop for each major version.
@@ -87,6 +89,9 @@ public abstract class ShimLoader {
}
+ private static final String SCHEDULER_SHIM_CLASSE =
+ "org.apache.hadoop.hive.schshim.FairSchedulerShim";
+
/**
* Factory method to get an instance of HadoopShims based on the
* version of Hadoop on the classpath.
@@ -124,6 +129,13 @@ public abstract class ShimLoader {
return hadoopThriftAuthBridge;
}
+ public static synchronized SchedulerShim getSchedulerShims() {
+ if (schedulerShim == null) {
+ schedulerShim = createShim(SCHEDULER_SHIM_CLASSE, SchedulerShim.class);
+ }
+ return schedulerShim;
+ }
+
private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) {
String vers = getMajorVersion();
String className = classMap.get(vers);
Modified: hive/trunk/shims/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/pom.xml?rev=1636225&r1=1636224&r2=1636225&view=diff
==============================================================================
--- hive/trunk/shims/pom.xml (original)
+++ hive/trunk/shims/pom.xml Mon Nov 3 00:49:40 2014
@@ -37,6 +37,7 @@
<module>common-secure</module>
<module>0.20S</module>
<module>0.23</module>
+ <module>scheduler</module>
<module>aggregator</module>
</modules>
</project>
Added: hive/trunk/shims/scheduler/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/scheduler/pom.xml?rev=1636225&view=auto
==============================================================================
--- hive/trunk/shims/scheduler/pom.xml (added)
+++ hive/trunk/shims/scheduler/pom.xml Mon Nov 3 00:49:40 2014
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive</artifactId>
+ <version>0.15.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-scheduler</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Shims Scheduler</name>
+
+ <properties>
+ <hive.path.to.root>../..</hive.path.to.root>
+ </properties>
+
+ <dependencies>
+ <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+ <!-- intra-project -->
+ <dependency>
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-common-secure</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- inter-project -->
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons-logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <version>${hadoop-23.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+</project>
Added: hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java?rev=1636225&view=auto
==============================================================================
--- hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java (added)
+++ hive/trunk/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java Mon Nov 3 00:49:40 2014
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.schshim;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.shims.SchedulerShim;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy;
+
+public class FairSchedulerShim implements SchedulerShim {
+ private static final Log LOG = LogFactory.getLog(FairSchedulerShim.class);
+ private static final String MR2_JOB_QUEUE_PROPERTY = "mapreduce.job.queuename";
+
+ @Override
+ public void refreshDefaultQueue(Configuration conf, String userName)
+ throws IOException {
+ String requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
+ final AtomicReference<AllocationConfiguration> allocConf = new AtomicReference<AllocationConfiguration>();
+
+ AllocationFileLoaderService allocsLoader = new AllocationFileLoaderService();
+ allocsLoader.init(conf);
+ allocsLoader.setReloadListener(new AllocationFileLoaderService.Listener() {
+ @Override
+ public void onReload(AllocationConfiguration allocs) {
+ allocConf.set(allocs);
+ }
+ });
+ try {
+ allocsLoader.reloadAllocations();
+ } catch (Exception ex) {
+ throw new IOException("Failed to load queue allocations", ex);
+ }
+ if (allocConf.get() == null) {
+ allocConf.set(new AllocationConfiguration(conf));
+ }
+ QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy();
+ if (queuePolicy != null) {
+ requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
+ if (StringUtils.isNotBlank(requestedQueue)) {
+ LOG.debug("Setting queue name to " + requestedQueue + " for user "
+ + userName);
+ conf.set(MR2_JOB_QUEUE_PROPERTY, requestedQueue);
+ }
+ }
+ }
+
+}