You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/09/29 02:35:38 UTC
[32/50] [abbrv] git commit: TEZ-1550.
TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows
(Prakash Ramachandran via bikas)
TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows (Prakash Ramachandran via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/97528e91
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/97528e91
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/97528e91
Branch: refs/heads/branch-0.5
Commit: 97528e9188a05b323699222e21ed7f22fd7ff8e3
Parents: 41fe736
Author: Bikas Saha <bi...@apache.org>
Authored: Sun Sep 21 13:46:58 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sun Sep 21 13:46:58 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tez/common/EnvironmentUpdateUtils.java | 127 +++++++++++++++++++
.../tez/dag/utils/EnvironmentUpdateUtils.java | 92 --------------
.../tez/common/TestEnvironmentUpdateUtils.java | 34 +++++
.../dag/utils/TestEnvironmentUpdateUtils.java | 34 -----
.../java/org/apache/tez/client/LocalClient.java | 2 +-
.../app/launcher/LocalContainerLauncher.java | 6 +-
.../library/output/TestOnFileSortedOutput.java | 2 +-
.../library/shuffle/common/TestFetcher.java | 2 +-
9 files changed, 171 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ab8b533..f069959 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -49,6 +49,8 @@ ALL CHANGES
TestTezCommonUtils for Windows
TEZ-1598. DAGClientTimelineImpl uses ReflectiveOperationException (which has JDK 1.7 dependency)
TEZ-1599. TezClient.preWarm() is not enabled
+ TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on
+ Windows
Release 0.5.0: 2014-09-03
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
new file mode 100644
index 0000000..cd519d9
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.Shell;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A utility class which allows one to dynamically update/change Environment variables
+ */
+@InterfaceAudience.Private
+public class EnvironmentUpdateUtils {
+
+ /**
+ * Allows dynamic update to the environment variables. After calling put,
+ * System.getenv(key) will then return value.
+ *
+ * @param key System environment variable
+ * @param value Value to assign to system environment variable
+ */
+ public static void put(String key, String value){
+ Map<String, String> environment = new HashMap<String, String>(System.getenv());
+ environment.put(key, value);
+ if (!Shell.WINDOWS) {
+ updateEnvironment(environment);
+ } else {
+ updateEnvironmentOnWindows(environment);
+ }
+ }
+
+ /**
+ * Allows dynamic update to a collection of environment variables. After
+ * calling putAll, System.getenv(key) will then return value for each entry
+ * in the map
+ *
+ * @param additionalEnvironment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ public static void putAll(Map<String, String> additionalEnvironment) {
+ Map<String, String> environment = new HashMap<String, String>(System.getenv());
+ environment.putAll(additionalEnvironment);
+ if (!Shell.WINDOWS) {
+ updateEnvironment(environment);
+ } else {
+ updateEnvironmentOnWindows(environment);
+ }
+ }
+
+ /**
+ * Finds and modifies internal storage for system environment variables using
+ * reflection
+ *
+ * @param environment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ @SuppressWarnings("unchecked")
+ private static void updateEnvironment(Map<String, String> environment) {
+ final Map<String, String> currentEnv = System.getenv();
+ copyMapValuesToPrivateField(currentEnv.getClass(), currentEnv, "m", environment);
+ }
+
+ /**
+ * Finds and modifies internal storage for system environment variables using reflection. This
+ * method works only on windows. Note that the actual env is not modified, rather the copy of env
+ * which the JVM creates at the beginning of execution is.
+ *
+ * @param environment Collection where the key is the System
+ * environment variable and the value is the value to assign the system
+ * environment variable
+ */
+ @SuppressWarnings("unchecked")
+ private static void updateEnvironmentOnWindows(Map<String, String> environment) {
+ try {
+ Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
+ copyMapValuesToPrivateField(processEnvironmentClass, null, "theEnvironment", environment);
+ copyMapValuesToPrivateField(processEnvironmentClass, null, "theCaseInsensitiveEnvironment",
+ environment);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ }
+ }
+
+ /**
+ * Copies the given map values to the field specified by {@code fieldName}
+ * @param klass The {@code Class} of the object
+ * @param object The object to modify or null if the field is static
+ * @param fieldName The name of the field to set
+ * @param newMapValues The values to replace the current map.
+ */
+ @SuppressWarnings("unchecked")
+ private static void copyMapValuesToPrivateField(Class<?> klass, Object object, String fieldName,
+ Map<String, String> newMapValues) {
+ try {
+ Field field = klass.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ Map<String, String> currentMap = (Map<String, String>) field.get(object);
+ currentMap.clear();
+ currentMap.putAll(newMapValues);
+ } catch (NoSuchFieldException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException("Failed to update Environment variables", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java b/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
deleted file mode 100644
index a8b4ced..0000000
--- a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.lang.reflect.Field;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A utility class which allows one to dynamically update/change Environment variables
- */
-@InterfaceAudience.Private
-public class EnvironmentUpdateUtils {
-
- /**
- * Allows dynamic update to the environment variables. After calling put,
- * System.getenv(key) will then return value.
- *
- * @param key System environment variable
- * @param value Value to assign to system environment variable
- */
- public static void put(String key, String value){
- Map<String, String> environment = new HashMap<String, String>(System.getenv());
- environment.put(key, value);
- updateEnvironment(environment);
- }
-
- /**
- * Allows dynamic update to a collection of environment variables. After
- * calling putAll, System.getenv(key) will then return value for each entry
- * in the map
- *
- * @param additionalEnvironment Collection where the key is the System
- * environment variable and the value is the value to assign the system
- * environment variable
- */
- public static void putAll(Map<String, String> additionalEnvironment) {
- Map<String, String> environment = new HashMap<String, String>(System.getenv());
- environment.putAll(additionalEnvironment);
- updateEnvironment(environment);
- }
-
- /**
- * Finds and modifies internal storage for system environment variables using
- * reflection
- *
- * @param environment Collection where the key is the System
- * environment variable and the value is the value to assign the system
- * environment variable
- */
- @SuppressWarnings("unchecked")
- private static void updateEnvironment(Map<String, String> environment) {
- try {
- Class<?>[] classes = Collections.class.getDeclaredClasses();
- for (Class<?> clazz : classes) {
- if ("java.util.Collections$UnmodifiableMap".equals(clazz.getName())) {
- Field field = clazz.getDeclaredField("m");
- field.setAccessible(true);
- Object obj = field.get(System.getenv());
- Map<String, String> map = (Map<String, String>)obj;
- map.clear();
- map.putAll(environment);
- }
- }
- }
- catch (NoSuchFieldException e) {
- throw new IllegalStateException("Failed to update Environment variables", e);
- }
- catch (IllegalAccessException e) {
- throw new IllegalStateException("Failed to update Environment variables", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
new file mode 100644
index 0000000..d8c6980
--- /dev/null
+++ b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestEnvironmentUpdateUtils {
+
+ @Test
+ public void testMultipleUpdateEnvironment() {
+ EnvironmentUpdateUtils.put("test.environment1", "test.value1");
+ EnvironmentUpdateUtils.put("test.environment2", "test.value2");
+ assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
+ assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java b/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
deleted file mode 100644
index d4a42c4..0000000
--- a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Test;
-
-public class TestEnvironmentUpdateUtils {
-
- @Test
- public void testMultipleUpdateEnvironment() {
- EnvironmentUpdateUtils.put("test.environment1", "test.value1");
- EnvironmentUpdateUtils.put("test.environment2", "test.value2");
- assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
- assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index 0a95cf0..bd7089f 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.log4j.Logger;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
@@ -53,7 +54,6 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.DAGAppMaster;
import org.apache.tez.dag.app.DAGAppMasterState;
import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7407544..baeb9a3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -46,8 +46,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.dag.api.TezConfiguration;
@@ -65,7 +67,6 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.task.TezChild;
@@ -111,6 +112,9 @@ public class LocalContainerLauncher extends AbstractService implements
Base64.encodeBase64String(ByteBuffer.allocate(4).putInt(0).array()));
EnvironmentUpdateUtils.put(Environment.NM_HOST.toString(),
InetAddress.getLocalHost().getHostName());
+ if (Shell.WINDOWS) {
+ EnvironmentUpdateUtils.put(Environment.USER.name(), System.getenv("USERNAME"));
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 8a7f2a6..f9a3dd5 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -25,11 +25,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
import org.apache.tez.runtime.api.OutputContext;
http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
index 9502555..2ef6ce6 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;