You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by de...@apache.org on 2019/03/29 18:01:16 UTC
[hadoop] branch trunk updated: YARN-9270. Minor cleanup in
TestFpgaDiscoverer. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
devaraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 56f1e13 YARN-9270. Minor cleanup in TestFpgaDiscoverer. Contributed by Peter Bacsko.
56f1e13 is described below
commit 56f1e131ecb9f53dcc2596807f8dec2d2b95bb51
Author: Devaraj K <de...@apache.org>
AuthorDate: Fri Mar 29 10:58:56 2019 -0700
YARN-9270. Minor cleanup in TestFpgaDiscoverer. Contributed by Peter Bacsko.
---
.../resources/fpga/FpgaResourceHandlerImpl.java | 12 +-
.../resourceplugin/fpga/FpgaDiscoverer.java | 30 +--
.../fpga/FpgaNodeResourceUpdateHandler.java | 13 +-
.../resourceplugin/fpga/FpgaResourcePlugin.java | 12 +-
.../resources/fpga/TestFpgaResourceHandler.java | 26 ++-
.../resourceplugin/fpga/TestFpgaDiscoverer.java | 211 ++++++++-------------
6 files changed, 124 insertions(+), 180 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
index cd1ea13..d06b805 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
@@ -64,6 +64,8 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
private final CGroupsHandler cGroupsHandler;
+ private final FpgaDiscoverer fpgaDiscoverer;
+
public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -72,10 +74,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
public FpgaResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor,
- AbstractFpgaVendorPlugin plugin) {
+ AbstractFpgaVendorPlugin plugin,
+ FpgaDiscoverer fpgaDiscoverer) {
this.allocator = new FpgaResourceAllocator(nmContext);
this.vendorPlugin = plugin;
- FpgaDiscoverer.getInstance().setResourceHanderPlugin(vendorPlugin);
+ this.fpgaDiscoverer = fpgaDiscoverer;
this.cGroupsHandler = cGroupsHandler;
this.privilegedOperationExecutor = privilegedOperationExecutor;
}
@@ -99,8 +102,7 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
}
LOG.info("FPGA Plugin bootstrap success.");
// Get avialable devices minor numbers from toolchain or static configuration
- List<FpgaResourceAllocator.FpgaDevice> fpgaDeviceList =
- FpgaDiscoverer.getInstance().discover();
+ List<FpgaDevice> fpgaDeviceList = fpgaDiscoverer.discover();
allocator.addFpgaDevices(vendorPlugin.getFpgaType(), fpgaDeviceList);
this.cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.DEVICES);
@@ -183,7 +185,7 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
" if you want YARN to program the device");
} else {
LOG.info("IP file path:" + ipFilePath);
- List<FpgaResourceAllocator.FpgaDevice> allowed = allocation.getAllowed();
+ List<FpgaDevice> allowed = allocation.getAllowed();
String majorMinorNumber;
for (int i = 0; i < allowed.size(); i++) {
FpgaDevice device = allowed.get(i);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
index a049038..185effa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.AoclOutputBasedDiscoveryStrategy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.FPGADiscoveryStrategy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.ScriptBasedFPGADiscoveryStrategy;
@@ -49,12 +50,8 @@ public class FpgaDiscoverer {
private static final Logger LOG = LoggerFactory.getLogger(
FpgaDiscoverer.class);
- private static FpgaDiscoverer instance;
-
private Configuration conf = null;
-
private AbstractFpgaVendorPlugin plugin = null;
-
private List<FpgaResourceAllocator.FpgaDevice> currentFpgaInfo = null;
private Function<String, Optional<String>> scriptRunner = this::runScript;
@@ -62,36 +59,17 @@ public class FpgaDiscoverer {
// shell command timeout
public static final int MAX_EXEC_TIMEOUT_MS = 10 * 1000;
- static {
- instance = new FpgaDiscoverer();
- }
-
- public static FpgaDiscoverer getInstance() {
- return instance;
- }
-
@VisibleForTesting
void setScriptRunner(Function<String, Optional<String>> scriptRunner) {
this.scriptRunner = scriptRunner;
}
@VisibleForTesting
- static void reset() {
- instance = new FpgaDiscoverer();
- }
-
- @VisibleForTesting
- public static FpgaDiscoverer setInstance(FpgaDiscoverer newInstance) {
- instance = newInstance;
- return instance;
- }
-
- @VisibleForTesting
public void setConf(Configuration configuration) {
this.conf = configuration;
}
- public List<FpgaResourceAllocator.FpgaDevice> getCurrentFpgaInfo() {
+ public List<FpgaDevice> getCurrentFpgaInfo() {
return currentFpgaInfo;
}
@@ -119,9 +97,9 @@ public class FpgaDiscoverer {
* @return the list of FPGA devices
* @throws ResourceHandlerException if there's any error during discovery
**/
- public List<FpgaResourceAllocator.FpgaDevice> discover()
+ public List<FpgaDevice> discover()
throws ResourceHandlerException {
- List<FpgaResourceAllocator.FpgaDevice> list;
+ List<FpgaDevice> list;
String allowed = this.conf.get(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES);
String availableDevices = conf.get(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaNodeResourceUpdateHandler.java
index fdba9a4..f39dada 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaNodeResourceUpdateHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaNodeResourceUpdateHandler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga;
-
import static org.apache.hadoop.yarn.api.records.ResourceInformation.FPGA_URI;
import java.util.LinkedList;
@@ -30,22 +29,28 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FpgaNodeResourceUpdateHandler extends NodeResourceUpdaterPlugin {
+ private final FpgaDiscoverer fpgaDiscoverer;
+
private static final Logger LOG = LoggerFactory.getLogger(
FpgaNodeResourceUpdateHandler.class);
+ public FpgaNodeResourceUpdateHandler(FpgaDiscoverer fpgaDiscoverer) {
+ this.fpgaDiscoverer = fpgaDiscoverer;
+ }
+
@Override
public void updateConfiguredResource(Resource res) throws YarnException {
LOG.info("Initializing configured FPGA resources for the NodeManager.");
- List<FpgaResourceAllocator.FpgaDevice> list = FpgaDiscoverer.getInstance().getCurrentFpgaInfo();
+ List<FpgaDevice> list = fpgaDiscoverer.getCurrentFpgaInfo();
List<Integer> minors = new LinkedList<>();
- for (FpgaResourceAllocator.FpgaDevice device : list) {
+ for (FpgaDevice device : list) {
minors.add(device.getMinor());
}
if (minors.isEmpty()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java
index 4dab1a4..c01d547 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java
@@ -44,6 +44,7 @@ public class FpgaResourcePlugin implements ResourcePlugin {
private AbstractFpgaVendorPlugin vendorPlugin = null;
private FpgaNodeResourceUpdateHandler fpgaNodeResourceUpdateHandler = null;
+ private FpgaDiscoverer fpgaDiscoverer;
private AbstractFpgaVendorPlugin createFpgaVendorPlugin(Configuration conf) {
String vendorPluginClass = conf.get(YarnConfiguration.NM_FPGA_VENDOR_PLUGIN,
@@ -68,9 +69,11 @@ public class FpgaResourcePlugin implements ResourcePlugin {
public void initialize(Context context) throws YarnException {
// Get vendor plugin from configuration
this.vendorPlugin = createFpgaVendorPlugin(context.getConf());
- FpgaDiscoverer.getInstance().setResourceHanderPlugin(vendorPlugin);
- FpgaDiscoverer.getInstance().initialize(context.getConf());
- fpgaNodeResourceUpdateHandler = new FpgaNodeResourceUpdateHandler();
+ fpgaDiscoverer = new FpgaDiscoverer();
+ fpgaDiscoverer.setResourceHanderPlugin(vendorPlugin);
+ fpgaDiscoverer.initialize(context.getConf());
+ fpgaNodeResourceUpdateHandler =
+ new FpgaNodeResourceUpdateHandler(fpgaDiscoverer);
}
@Override
@@ -79,7 +82,8 @@ public class FpgaResourcePlugin implements ResourcePlugin {
PrivilegedOperationExecutor privilegedOperationExecutor) {
if (fpgaResourceHandler == null) {
fpgaResourceHandler = new FpgaResourceHandlerImpl(nmContext,
- cGroupsHandler, privilegedOperationExecutor, vendorPlugin);
+ cGroupsHandler, privilegedOperationExecutor, vendorPlugin,
+ fpgaDiscoverer);
}
return fpgaResourceHandler;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/TestFpgaResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/TestFpgaResourceHandler.java
index 9564c72..0b7c3566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/TestFpgaResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/TestFpgaResourceHandler.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@@ -91,16 +92,18 @@ public class TestFpgaResourceHandler {
private ConcurrentHashMap<ContainerId, Container> runningContainersMap;
private IntelFpgaOpenclPlugin mockVendorPlugin;
private List<FpgaDevice> deviceList;
+ private FpgaDiscoverer fpgaDiscoverer;
private static final String vendorType = "IntelOpenCL";
private File dummyAocx;
private String getTestParentFolder() {
- File f = new File("target/temp/" + TestFpgaResourceHandler.class.getName());
+ File f = new File("target/temp/" +
+ TestFpgaResourceHandler.class.getName());
return f.getAbsolutePath();
}
@Before
- public void setup() throws IOException {
+ public void setup() throws IOException, YarnException {
CustomResourceTypesConfigurationProvider.
initResourceTypes(ResourceInformation.FPGA_URI);
configuration = new YarnConfiguration();
@@ -116,13 +119,16 @@ public class TestFpgaResourceHandler {
}
String aocxPath = getTestParentFolder() + "/test.aocx";
mockVendorPlugin = mockPlugin(vendorType, deviceList, aocxPath);
- FpgaDiscoverer.getInstance().setConf(configuration);
+ fpgaDiscoverer = new FpgaDiscoverer();
+ fpgaDiscoverer.setResourceHanderPlugin(mockVendorPlugin);
+ fpgaDiscoverer.initialize(configuration);
when(mockContext.getNMStateStore()).thenReturn(mockNMStateStore);
runningContainersMap = new ConcurrentHashMap<>();
when(mockContext.getContainers()).thenReturn(runningContainersMap);
fpgaResourceHandler = new FpgaResourceHandlerImpl(mockContext,
- mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin);
+ mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin,
+ fpgaDiscoverer);
dummyAocx = new File(aocxPath);
Files.createParentDirs(dummyAocx);
@@ -143,7 +149,8 @@ public class TestFpgaResourceHandler {
String allowed = "auto";
configuration.set(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES, allowed);
fpgaResourceHandler.bootstrap(configuration);
- verify(mockVendorPlugin, times(1)).initPlugin(configuration);
+ // initPlugin() was also called in setup()
+ verify(mockVendorPlugin, times(2)).initPlugin(configuration);
verify(mockCGroupsHandler, times(1)).initializeCGroupController(
CGroupsHandler.CGroupController.DEVICES);
Assert.assertEquals(5, fpgaResourceHandler.getFpgaAllocator()
@@ -152,7 +159,8 @@ public class TestFpgaResourceHandler {
.getAllowedFpga().size());
// Case 2. subset of devices
fpgaResourceHandler = new FpgaResourceHandlerImpl(mockContext,
- mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin);
+ mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin,
+ fpgaDiscoverer);
allowed = "0,1,2";
configuration.set(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES, allowed);
fpgaResourceHandler.bootstrap(configuration);
@@ -174,7 +182,8 @@ public class TestFpgaResourceHandler {
// Case 3. User configuration contains invalid minor device number
fpgaResourceHandler = new FpgaResourceHandlerImpl(mockContext,
- mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin);
+ mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin,
+ fpgaDiscoverer);
allowed = "0,1,7";
configuration.set(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES, allowed);
fpgaResourceHandler.bootstrap(configuration);
@@ -509,7 +518,8 @@ public class TestFpgaResourceHandler {
mockVendorPlugin =
mockPlugin(vendorType, deviceList, dummyAocx.getAbsolutePath());
fpgaResourceHandler = new FpgaResourceHandlerImpl(mockContext,
- mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin);
+ mockCGroupsHandler, mockPrivilegedExecutor, mockVendorPlugin,
+ fpgaDiscoverer);
fpgaResourceHandler.bootstrap(configuration);
fpgaResourceHandler.preStart(mockContainer(0, 1, "GEMM"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
index 1eb2431..92e9db2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
@@ -28,8 +28,6 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.lang.reflect.Field;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,6 +39,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -50,6 +49,11 @@ public class TestFpgaDiscoverer {
@Rule
public ExpectedException expected = ExpectedException.none();
+ private File fakeBinary;
+ private IntelFpgaOpenclPlugin openclPlugin;
+ private Configuration conf;
+ private FpgaDiscoverer fpgaDiscoverer;
+
private String getTestParentFolder() {
File f = new File("target/temp/" + TestFpgaDiscoverer.class.getName());
return f.getAbsolutePath();
@@ -65,110 +69,97 @@ public class TestFpgaDiscoverer {
File f = new File(folder);
FileUtils.deleteDirectory(f);
f.mkdirs();
- FpgaDiscoverer.reset();
+
+ conf = new Configuration();
+
+ openclPlugin = new IntelFpgaOpenclPlugin();
+ openclPlugin.initPlugin(conf);
+ openclPlugin.setInnerShellExecutor(mockPuginShell());
+
+ fpgaDiscoverer = new FpgaDiscoverer();
+ fpgaDiscoverer.setResourceHanderPlugin(openclPlugin);
}
- // A dirty hack to modify the env of the current JVM itself - Dirty, but
- // should be okay for testing.
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private static void setNewEnvironmentHack(Map<String, String> newenv)
- throws Exception {
- try {
- Class<?> cl = Class.forName("java.lang.ProcessEnvironment");
- Field field = cl.getDeclaredField("theEnvironment");
- field.setAccessible(true);
- Map<String, String> env = (Map<String, String>) field.get(null);
- env.clear();
- env.putAll(newenv);
- Field ciField = cl.getDeclaredField("theCaseInsensitiveEnvironment");
- ciField.setAccessible(true);
- Map<String, String> cienv = (Map<String, String>) ciField.get(null);
- cienv.clear();
- cienv.putAll(newenv);
- } catch (NoSuchFieldException e) {
- Class[] classes = Collections.class.getDeclaredClasses();
- Map<String, String> env = System.getenv();
- for (Class cl : classes) {
- if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
- Field field = cl.getDeclaredField("m");
- field.setAccessible(true);
- Object obj = field.get(env);
- Map<String, String> map = (Map<String, String>) obj;
- map.clear();
- map.putAll(newenv);
- }
- }
+ @After
+ public void afterTest() {
+ if (fakeBinary != null) {
+ fakeBinary.delete();
}
}
@Test
- public void testLinuxFpgaResourceDiscoverPluginConfig() throws Exception {
- Configuration conf = new Configuration(false);
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
-
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- // because FPGA discoverer is a singleton, we use setPlugin to make
- // FpgaDiscoverer.getInstance().diagnose() work in openclPlugin.initPlugin()
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
+ public void testExecutablePathWithoutExplicitConfig()
+ throws YarnException {
+ fpgaDiscoverer.initialize(conf);
- discoverer.initialize(conf);
- // Case 1. No configuration set for binary(no environment "ALTERAOCLSDKROOT" set)
assertEquals("No configuration(no environment ALTERAOCLSDKROOT set)" +
- "should return just a single binary name",
+ " should return just a single binary name",
"aocl", openclPlugin.getPathToExecutable());
+ }
- // Case 2. With correct configuration and file exists
- File fakeBinary = new File(getTestParentFolder() + "/aocl");
- conf.set(YarnConfiguration.NM_FPGA_PATH_TO_EXEC, getTestParentFolder() + "/aocl");
+ @Test
+ public void testExecutablePathWithCorrectConfig()
+ throws IOException, YarnException {
+ fakeBinary = new File(getTestParentFolder() + "/aocl");
+ conf.set(YarnConfiguration.NM_FPGA_PATH_TO_EXEC,
+ getTestParentFolder() + "/aocl");
touchFile(fakeBinary);
- discoverer.initialize(conf);
+
+ fpgaDiscoverer.initialize(conf);
+
assertEquals("Correct configuration should return user setting",
getTestParentFolder() + "/aocl", openclPlugin.getPathToExecutable());
+ }
+
+ @Test
+ public void testExecutablePathWhenFileDoesNotExist()
+ throws YarnException {
+ conf.set(YarnConfiguration.NM_FPGA_PATH_TO_EXEC,
+ getTestParentFolder() + "/aocl");
- // Case 3. With correct configuration but file doesn't exists. Use default
- fakeBinary.delete();
- discoverer.initialize(conf);
- assertEquals("Should return just a single binary name",
+ fpgaDiscoverer.initialize(conf);
+
+ assertEquals("File doesn't exists - expected a single binary name",
"aocl", openclPlugin.getPathToExecutable());
+ }
- // Case 4. Set a empty value
+ @Test
+ public void testExecutablePathWhenFileIsEmpty()
+ throws YarnException {
conf.set(YarnConfiguration.NM_FPGA_PATH_TO_EXEC, "");
- discoverer.initialize(conf);
+
+ fpgaDiscoverer.initialize(conf);
+
assertEquals("configuration with empty string value, should use aocl",
"aocl", openclPlugin.getPathToExecutable());
+ }
- // Case 5. No configuration set for binary, but set environment "ALTERAOCLSDKROOT"
- // we load the default configuration to start with
- conf = new Configuration(true);
+ @Test
+ public void testExecutablePathWithSdkRootSet()
+ throws IOException, YarnException {
fakeBinary = new File(getTestParentFolder() + "/bin/aocl");
fakeBinary.getParentFile().mkdirs();
touchFile(fakeBinary);
Map<String, String> newEnv = new HashMap<String, String>();
newEnv.put("ALTERAOCLSDKROOT", getTestParentFolder());
- setNewEnvironmentHack(newEnv);
- discoverer.initialize(conf);
+ openclPlugin.setEnvProvider(s -> {
+ return newEnv.get(s); });
+
+ fpgaDiscoverer.initialize(conf);
+
assertEquals("No configuration but with environment ALTERAOCLSDKROOT set",
getTestParentFolder() + "/bin/aocl", openclPlugin.getPathToExecutable());
-
}
@Test
public void testDiscoveryWhenAvailableDevicesDefined()
throws YarnException {
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_AVAILABLE_DEVICES,
"acl0/243:0,acl1/244:1");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
+ fpgaDiscoverer.initialize(conf);
+ List<FpgaDevice> devices = fpgaDiscoverer.discover();
- discoverer.initialize(conf);
- List<FpgaDevice> devices = discoverer.discover();
assertEquals("Number of devices", 2, devices.size());
FpgaDevice device0 = devices.get(0);
FpgaDevice device1 = devices.get(1);
@@ -188,18 +179,11 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("No FPGA devices were specified");
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_AVAILABLE_DEVICES,
"");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
-
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
}
@Test
@@ -208,37 +192,24 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("Illegal device specification string");
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_AVAILABLE_DEVICES,
"illegal/243:0,acl1/244=1");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
-
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
}
@Test
public void testDiscoveryWhenExternalScriptDefined()
throws YarnException {
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT,
"/dummy/script");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
- discoverer.setScriptRunner(s -> {
+ fpgaDiscoverer.setScriptRunner(s -> {
return Optional.of("acl0/243:0,acl1/244:1"); });
+ fpgaDiscoverer.initialize(conf);
+ List<FpgaDevice> devices = fpgaDiscoverer.discover();
- discoverer.initialize(conf);
- List<FpgaDevice> devices = discoverer.discover();
assertEquals("Number of devices", 2, devices.size());
FpgaDevice device0 = devices.get(0);
FpgaDevice device1 = devices.get(1);
@@ -258,20 +229,14 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("No FPGA devices were specified");
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT,
"/dummy/script");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
- discoverer.setScriptRunner(s -> {
+ fpgaDiscoverer.setScriptRunner(s -> {
return Optional.of(""); });
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
}
@Test
@@ -280,20 +245,14 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("Unable to run external script");
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT,
"/dummy/script");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
- discoverer.setScriptRunner(s -> {
+ fpgaDiscoverer.setScriptRunner(s -> {
return Optional.empty(); });
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
}
@Test
@@ -302,17 +261,10 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("Unable to run external script");
- Configuration conf = new Configuration(false);
conf.set(YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT, "");
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
-
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
}
@Test
@@ -323,21 +275,14 @@ public class TestFpgaDiscoverer {
expected.expect(ResourceHandlerException.class);
expected.expectMessage("Unable to run external script");
- Configuration conf = new Configuration(false);
fakeScript = new File(getTestParentFolder() + "/fakeScript");
touchFile(fakeScript);
fakeScript.setExecutable(false);
conf.set(YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT,
fakeScript.getAbsolutePath());
- FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance();
-
- IntelFpgaOpenclPlugin openclPlugin = new IntelFpgaOpenclPlugin();
- discoverer.setResourceHanderPlugin(openclPlugin);
- openclPlugin.initPlugin(conf);
- openclPlugin.setInnerShellExecutor(mockPuginShell());
- discoverer.initialize(conf);
- discoverer.discover();
+ fpgaDiscoverer.initialize(conf);
+ fpgaDiscoverer.discover();
} finally {
fakeScript.delete();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org