You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/09/22 07:22:27 UTC
[1/2] git commit: TEZ-1550.
TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows
(Prakash Ramachandran via bikas)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 f67dcd794 -> 375863f49
TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows (Prakash Ramachandran via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2b20bd79
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2b20bd79
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2b20bd79
Branch: refs/heads/branch-0.5
Commit: 2b20bd7928f0f4a11291a48e6991f19a130d3b01
Parents: f67dcd7
Author: Bikas Saha <bi...@apache.org>
Authored: Sun Sep 21 13:46:58 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sun Sep 21 13:48:49 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tez/common/EnvironmentUpdateUtils.java | 127 +++++++++++++++++++
.../tez/dag/utils/EnvironmentUpdateUtils.java | 92 --------------
.../tez/common/TestEnvironmentUpdateUtils.java | 34 +++++
.../dag/utils/TestEnvironmentUpdateUtils.java | 34 -----
.../java/org/apache/tez/client/LocalClient.java | 2 +-
.../app/launcher/LocalContainerLauncher.java | 6 +-
.../library/output/TestOnFileSortedOutput.java | 2 +-
.../library/shuffle/common/TestFetcher.java | 2 +-
9 files changed, 171 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 05ca588..a127cc5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -30,6 +30,8 @@ ALL CHANGES
TestTezCommonUtils for Windows
TEZ-1598. DAGClientTimelineImpl uses ReflectiveOperationException (which has JDK 1.7 dependency)
TEZ-1599. TezClient.preWarm() is not enabled
+ TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on
+ Windows
Release 0.5.0: 2014-09-03
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
new file mode 100644
index 0000000..cd519d9
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.Shell;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A utility class which allows one to dynamically update/change Environment variables
+ */
+@InterfaceAudience.Private
+public class EnvironmentUpdateUtils {
+
+ /**
+ * Allows dynamic update to the environment variables. After calling put,
+ * System.getenv(key) will then return value.
+ *
+ * @param key System environment variable
+ * @param value Value to assign to system environment variable
+ */
+ public static void put(String key, String value){
+ Map<String, String> environment = new HashMap<String, String>(System.getenv());
+ environment.put(key, value);
+ if (!Shell.WINDOWS) {
+ updateEnvironment(environment);
+ } else {
+ updateEnvironmentOnWindows(environment);
+ }
+ }
+
+ /**
+ * Allows dynamic update to a collection of environment variables. After
+ * calling putAll, System.getenv(key) will then return value for each entry
+ * in the map
+ *
+ * @param additionalEnvironment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ public static void putAll(Map<String, String> additionalEnvironment) {
+ Map<String, String> environment = new HashMap<String, String>(System.getenv());
+ environment.putAll(additionalEnvironment);
+ if (!Shell.WINDOWS) {
+ updateEnvironment(environment);
+ } else {
+ updateEnvironmentOnWindows(environment);
+ }
+ }
+
+ /**
+ * Finds and modifies internal storage for system environment variables using
+ * reflection
+ *
+ * @param environment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ @SuppressWarnings("unchecked")
+ private static void updateEnvironment(Map<String, String> environment) {
+ final Map<String, String> currentEnv = System.getenv();
+ copyMapValuesToPrivateField(currentEnv.getClass(), currentEnv, "m", environment);
+ }
+
+ /**
+ * Finds and modifies internal storage for system environment variables using reflection. This
+ * method works only on windows. Note that the actual env is not modified, rather the copy of env
+ * which the JVM creates at the beginning of execution is.
+ *
+ * @param environment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ @SuppressWarnings("unchecked")
+ private static void updateEnvironmentOnWindows(Map<String, String> environment) {
+ try {
+ Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
+ copyMapValuesToPrivateField(processEnvironmentClass, null, "theEnvironment", environment);
+ copyMapValuesToPrivateField(processEnvironmentClass, null, "theCaseInsensitiveEnvironment",
+ environment);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ }
+ }
+
+ /**
+ * Copies the given map values to the field specified by {@code fieldName}
+ * @param klass The {@code Class} of the object
+ * @param object The object to modify or null if the field is static
+ * @param fieldName The name of the field to set
+ * @param newMapValues The values to replace the current map.
+ */
+ @SuppressWarnings("unchecked")
+ private static void copyMapValuesToPrivateField(Class<?> klass, Object object, String fieldName,
+ Map<String, String> newMapValues) {
+ try {
+ Field field = klass.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ Map<String, String> currentMap = (Map<String, String>) field.get(object);
+ currentMap.clear();
+ currentMap.putAll(newMapValues);
+ } catch (NoSuchFieldException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java b/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
deleted file mode 100644
index a8b4ced..0000000
--- a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.lang.reflect.Field;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A utility class which allows one to dynamically update/change Environment variables
- */
-@InterfaceAudience.Private
-public class EnvironmentUpdateUtils {
-
- /**
- * Allows dynamic update to the environment variables. After calling put,
- * System.getenv(key) will then return value.
- *
- * @param key System environment variable
- * @param value Value to assign to system environment variable
- */
- public static void put(String key, String value){
- Map<String, String> environment = new HashMap<String, String>(System.getenv());
- environment.put(key, value);
- updateEnvironment(environment);
- }
-
- /**
- * Allows dynamic update to a collection of environment variables. After
- * calling putAll, System.getenv(key) will then return value for each entry
- * in the map
- *
- * @param additionalEnvironment Collection where the key is the System
- * environment variable and the value is the value to assign the system
- * environment variable
- */
- public static void putAll(Map<String, String> additionalEnvironment) {
- Map<String, String> environment = new HashMap<String, String>(System.getenv());
- environment.putAll(additionalEnvironment);
- updateEnvironment(environment);
- }
-
- /**
- * Finds and modifies internal storage for system environment variables using
- * reflection
- *
- * @param environment Collection where the key is the System
- * environment variable and the value is the value to assign the system
- * environment variable
- */
- @SuppressWarnings("unchecked")
- private static void updateEnvironment(Map<String, String> environment) {
- try {
- Class<?>[] classes = Collections.class.getDeclaredClasses();
- for (Class<?> clazz : classes) {
- if ("java.util.Collections$UnmodifiableMap".equals(clazz.getName())) {
- Field field = clazz.getDeclaredField("m");
- field.setAccessible(true);
- Object obj = field.get(System.getenv());
- Map<String, String> map = (Map<String, String>)obj;
- map.clear();
- map.putAll(environment);
- }
- }
- }
- catch (NoSuchFieldException e) {
- throw new IllegalStateException("Failed to update Environment variables", e);
- }
- catch (IllegalAccessException e) {
- throw new IllegalStateException("Failed to update Environment variables", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
new file mode 100644
index 0000000..d8c6980
--- /dev/null
+++ b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestEnvironmentUpdateUtils {
+
+ @Test
+ public void testMultipleUpdateEnvironment() {
+ EnvironmentUpdateUtils.put("test.environment1", "test.value1");
+ EnvironmentUpdateUtils.put("test.environment2", "test.value2");
+ assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
+ assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java b/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
deleted file mode 100644
index d4a42c4..0000000
--- a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Test;
-
-public class TestEnvironmentUpdateUtils {
-
- @Test
- public void testMultipleUpdateEnvironment() {
- EnvironmentUpdateUtils.put("test.environment1", "test.value1");
- EnvironmentUpdateUtils.put("test.environment2", "test.value2");
- assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
- assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index 0b615fa..1ce6c55 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.log4j.Logger;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
@@ -52,7 +53,6 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.DAGAppMaster;
import org.apache.tez.dag.app.DAGAppMasterState;
import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
public class LocalClient extends FrameworkClient {
public static final Logger LOG = Logger.getLogger(LocalClient.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7407544..baeb9a3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -46,8 +46,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.dag.api.TezConfiguration;
@@ -65,7 +67,6 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.task.TezChild;
@@ -111,6 +112,9 @@ public class LocalContainerLauncher extends AbstractService implements
Base64.encodeBase64String(ByteBuffer.allocate(4).putInt(0).array()));
EnvironmentUpdateUtils.put(Environment.NM_HOST.toString(),
InetAddress.getLocalHost().getHostName());
+ if (Shell.WINDOWS) {
+ EnvironmentUpdateUtils.put(Environment.USER.name(), System.getenv("USERNAME"));
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 8a7f2a6..f9a3dd5 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -25,11 +25,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
import org.apache.tez.runtime.api.OutputContext;
http://git-wip-us.apache.org/repos/asf/tez/blob/2b20bd79/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
index 9502555..2ef6ce6 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
[2/2] git commit: TEZ-1554. Failing tests in TestMRHelpers related to
environment on Windows (Prakash Ramachandran via bikas)
Posted by bi...@apache.org.
TEZ-1554. Failing tests in TestMRHelpers related to environment on Windows (Prakash Ramachandran via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/375863f4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/375863f4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/375863f4
Branch: refs/heads/branch-0.5
Commit: 375863f497219fb7e52c1afcf8225ad321ee6786
Parents: 2b20bd7
Author: Bikas Saha <bi...@apache.org>
Authored: Sun Sep 21 22:20:55 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sun Sep 21 22:22:08 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/mapreduce/hadoop/TestMRHelpers.java | 20 +++++++++++++++++---
2 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/375863f4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a127cc5..4ba38b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -32,6 +32,7 @@ ALL CHANGES
TEZ-1599. TezClient.preWarm() is not enabled
TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on
Windows
+ TEZ-1554. Failing tests in TestMRHelpers related to environment on Windows
Release 0.5.0: 2014-09-03
http://git-wip-us.apache.org/repos/asf/tez/blob/375863f4/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
index a405cba..27f1863 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TezConstants;
@@ -113,7 +114,11 @@ public class TestMRHelpers {
conf.set(MRJobConfig.REDUCE_ENV, "foo=red1,bar=red2");
conf.set(MRJobConfig.MAP_LOG_LEVEL, "TRACE");
conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "FATAL");
- conf.set(MRJobConfig.MAPRED_ADMIN_USER_ENV, "LD_LIBRARY_PATH=$TEZ_ADMIN_ENV_TEST/lib/native");
+ final String mapredAdminUserEnv = Shell.WINDOWS ?
+ "PATH=%PATH%" + File.pathSeparator + "%TEZ_ADMIN_ENV%\\bin":
+ "LD_LIBRARY_PATH=$TEZ_ADMIN_ENV_TEST/lib/native";
+
+ conf.set(MRJobConfig.MAPRED_ADMIN_USER_ENV, mapredAdminUserEnv);
return conf;
}
@@ -123,8 +128,17 @@ public class TestMRHelpers {
Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
- Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
- env.get(Environment.LD_LIBRARY_PATH.name()));
+
+ /* On non-windows platform ensure that LD_LIBRARY_PATH is being set and PWD is present.
+ * on windows platform LD_LIBRARY_PATH is not applicable. check the PATH is being appended
+ * by the user setting (ex user may set HADOOP_HOME\\bin.
+ */
+ if (!Shell.WINDOWS) {
+ Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
+ env.get(Environment.LD_LIBRARY_PATH.name()));
+ } else {
+ Assert.assertTrue(env.get(Environment.PATH.name()).contains(";%TEZ_ADMIN_ENV%\\bin"));
+ }
// TEZ-273 will reinstate this or similar.
// for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {