You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:43 UTC
[38/50] [abbrv] oozie git commit: OOZIE-2803 Mask passwords when
printing out configs/args in MapReduceMain and SparkMain (pbacsko via
rkanter)
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ccbf692d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ccbf692d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ccbf692d
Branch: refs/heads/oya
Commit: ccbf692d90f8e52ab30afcdc7e411209f3d2b94b
Parents: 77817e7
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Feb 24 13:18:11 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Feb 24 13:18:11 2017 -0800
----------------------------------------------------------------------
.../org/apache/oozie/util/Instrumentation.java | 1 +
.../org/apache/oozie/util/PasswordMasker.java | 121 ----------------
.../apache/oozie/util/TestPasswordMasker.java | 92 ------------
.../test/resources/instrumentation-os-env.json | 47 ------
.../instrumentation-system-properties.json | 88 ------------
release-log.txt | 1 +
.../oozie/action/hadoop/MapReduceMain.java | 13 +-
.../oozie/action/hadoop/PasswordMasker.java | 144 +++++++++++++++++++
.../oozie/action/hadoop/TestPasswordMasker.java | 140 ++++++++++++++++++
.../test/resources/instrumentation-os-env.json | 47 ++++++
.../instrumentation-system-properties.json | 88 ++++++++++++
.../apache/oozie/action/hadoop/SparkMain.java | 3 +-
12 files changed, 434 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/main/java/org/apache/oozie/util/Instrumentation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
index 55e00d4..45219a9 100644
--- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java
+++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
@@ -20,6 +20,7 @@ package org.apache.oozie.util;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.action.hadoop.PasswordMasker;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.Services;
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java b/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
deleted file mode 100644
index 1f8a0ab..0000000
--- a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.util;
-
-import com.google.common.collect.Maps;
-
-import javax.annotation.Nullable;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys.
- * <p/>
- * Tested with {@see System#getProperties()} and {@see System#getenv()}.
- */
-class PasswordMasker {
-
- /**
- * The mask that is applied to recognized passwords.
- **/
- private static final String PASSWORD_MASK = "*****";
-
- /**
- * A key is considered a password key, if it contains {{pass}}, case ignored.
- **/
- private static final String PASSWORD_KEY = "pass";
-
- /**
- * Tells us whether an OS environment variable that contains a password fragment.
- * <p/>
- * E.g. {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}.
- **/
- private static final String REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE =
- ".*[((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)]+.*";
-
- /**
- * Extracts a password fragment from an OS environment variable. Can be used iteratively to get all fragments.
- * <p/>
- * E.g. {{-Doozie.https.keystore.pass=password}} and {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}.
- * {@see java.util.Matcher#find()}
- **/
- private static final String REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE =
- "((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)";
-
- private static final Pattern PATTERN_CONTAINING_PASSWORD_FRAGMENTS = Pattern
- .compile(REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE);
-
- private static final Pattern PATTERN_EXTRACTING_PASSWORD_FRAGMENTS = Pattern
- .compile(REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE);
-
- Map<String, String> mask(Map<String, String> unmasked) {
- return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() {
- @Override
- public String transformEntry(@Nullable String key, @Nullable String value) {
- checkNotNull(key, "key has to be set");
- checkNotNull(value, "value has to be set");
-
- if (isPasswordKey(key)) {
- return PASSWORD_MASK;
- }
-
- if (containsPasswordFragment(value)) {
- return maskPasswordFragments(value);
- }
-
- return value;
- }
- });
- }
-
- private boolean isPasswordKey(String key) {
- return key.toLowerCase().contains(PASSWORD_KEY);
-
- }
-
- private boolean containsPasswordFragment(String maybePasswordFragments) {
- return PATTERN_CONTAINING_PASSWORD_FRAGMENTS
- .matcher(maybePasswordFragments)
- .matches();
- }
-
- private String maskPasswordFragments(String maybePasswordFragments) {
- StringBuilder maskedBuilder = new StringBuilder();
- Matcher passwordFragmentsMatcher = PATTERN_EXTRACTING_PASSWORD_FRAGMENTS
- .matcher(maybePasswordFragments);
-
- int start = 0, end;
- while (passwordFragmentsMatcher.find()) {
- end = passwordFragmentsMatcher.start();
-
- maskedBuilder.append(maybePasswordFragments.substring(start, end));
- maskedBuilder.append(passwordFragmentsMatcher.group(1));
- maskedBuilder.append(PASSWORD_MASK);
-
- start = passwordFragmentsMatcher.end();
- }
-
- maskedBuilder.append(maybePasswordFragments.substring(start));
-
- return maskedBuilder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java b/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
deleted file mode 100644
index b00cce7..0000000
--- a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.util;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestPasswordMasker {
-
- @Test
- public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception {
- Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-system-properties.json"));
-
- assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword");
- assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass");
- }
-
- @Test
- public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception {
- Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-os-env.json"));
-
- assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD");
- assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD");
- assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD");
-
- assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass=");
- assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword=");
-
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m");
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore");
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path=");
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, String> jsonToMap(String jsonPath) throws IOException {
- return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class);
- }
-
- private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) {
- assertEquals(String.format("Value of key '%s' should be masked.", passwordKey),
- "*****",
- mapContainingMaskedPassword.get(passwordKey));
- }
-
- private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey,
- String passwordFragmentKey) {
- assertEquals(
- String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.",
- passwordKey,
- passwordFragmentKey),
- "*****",
- getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey));
- }
-
- private String getFragmentValue(String base, String fragmentKey) {
- for (String fragment : base.split(" ")) {
- if (fragment.startsWith(fragmentKey)) {
- return fragment.substring(fragmentKey.length());
- }
- }
-
- return null;
- }
-
- private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) {
- assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment),
- masked.get(key).contains(valueFragment));
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/resources/instrumentation-os-env.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/instrumentation-os-env.json b/core/src/test/resources/instrumentation-os-env.json
deleted file mode 100644
index e85cd8d..0000000
--- a/core/src/test/resources/instrumentation-os-env.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "HADOOP_CREDSTORE_PASSWORD": "password",
- "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password",
- "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password",
- "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
- "HISTCONTROL": "ignoreboth",
- "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
- "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid",
- "MC_SID": "8597",
- "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local",
- "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local",
- "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home",
- "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out",
- "TERM": "xterm-256color",
- "LANG": "en_US.UTF-8",
- "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "OOZIE_CONFIG_FILE": "oozie-site.xml",
- "LOGNAME": "forsage",
- "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "XPC_SERVICE_NAME": "0",
- "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "TERM_PROGRAM_VERSION": "361.1",
- "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap",
- "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java",
- "SHELL": "/bin/bash",
- "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
- "TERM_PROGRAM": "Apple_Terminal",
- "OOZIE_ADMIN_PORT": "11001",
- "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users
/forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=",
- "USER": "forsage",
- "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
- "OOZIE_LOG4J_RELOAD": "10",
- "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/",
- "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners",
- "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage",
- "XPC_FLAGS": "0x0",
- "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie",
- "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C",
- "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore",
- "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0",
- "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render",
- "OOZIE_HTTP_PORT": "11000",
- "OOZIE_HTTPS_PORT": "11443",
- "SHLVL": "3",
- "HOME": "/Users/forsage",
- "OOZIE_LOG4J_FILE": "oozie-log4j.properties"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/resources/instrumentation-system-properties.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/instrumentation-system-properties.json b/core/src/test/resources/instrumentation-system-properties.json
deleted file mode 100644
index 61430d2..0000000
--- a/core/src/test/resources/instrumentation-system-properties.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{
- "javax.net.ssl.trustStorePassword": "password",
- "oozie.https.keystore.pass": "password",
- "gopherProxySet": "false",
- "awt.toolkit": "sun.lwawt.macosx.LWCToolkit",
- "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie",
- "file.encoding.pkg": "sun.io",
- "java.specification.version": "1.8",
- "sun.cpu.isalist": "",
- "sun.jnu.encoding": "UTF-8",
- "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar",
- "java.vm.vendor": "Oracle Corporation",
- "sun.arch.data.model": "64",
- "sun.font.fontmanager": "sun.font.CFontManager",
- "catalina.useNaming": "true",
- "java.vendor.url": "http://java.oracle.com/",
- "user.timezone": "Europe/Budapest",
- "os.name": "Mac OS X",
- "java.vm.specification.version": "1.8",
- "oozie.http.hostname": "Budapests-MacBook-Pro.local",
- "oozie.instance.id": "Budapests-MacBook-Pro.local",
- "sun.java.launcher": "SUN_STANDARD",
- "user.country": "US",
- "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
- "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib",
- "sun.java.command": "org.apache.catalina.startup.Bootstrap start",
- "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
- "sun.cpu.endian": "little",
- "user.home": "/Users/forsage",
- "user.language": "en",
- "java.specification.vendor": "Oracle Corporation",
- "java.naming.factory.url.pkgs": "org.apache.naming",
- "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre",
- "oozie.config.file": "oozie-site.xml",
- "oozie.log4j.reload": "10",
- "file.separator": "/",
- "oozie.https.keystore.file": "/Users/forsage/.keystore",
- "line.separator": "\n",
- "java.vm.specification.vendor": "Oracle Corporation",
- "java.specification.name": "Java Platform API Specification",
- "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log",
- "oozie.log4j.file": "oozie-log4j.properties",
- "oozie.admin.port": "11001",
- "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment",
- "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.",
- "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.",
- "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes",
- "server.loader": "",
- "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties",
- "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
- "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
- "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
- "java.runtime.version": "1.8.0_102-b14",
- "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory",
- "user.name": "forsage",
- "oozie.https.port": "11443",
- "path.separator": ":",
- "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar",
- "os.version": "10.11.6",
- "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed",
- "java.runtime.name": "Java(TM) SE Runtime Environment",
- "file.encoding": "UTF-8",
- "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM",
- "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/",
- "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp",
- "oozie.http.port": "11000",
- "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "java.version": "1.8.0_102",
- "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
- "os.arch": "x86_64",
- "java.vm.specification.name": "Java Virtual Machine Specification",
- "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob",
- "sun.os.patch.level": "unknown",
- "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "shared.loader": "",
- "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager",
- "java.library.path": "",
- "java.vendor": "Oracle Corporation",
- "java.vm.info": "mixed mode",
- "java.vm.version": "25.102-b14",
- "sun.io.unicode.encoding": "UnicodeBig",
- "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java",
- "tomcat.util.buf.StringCache.byte.enabled": "true",
- "java.class.version": "52.0",
- "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index df586c9..fdf6f2b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 23447cf..d376057 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import java.util.HashSet;
+
+import java.util.ArrayList;
import java.util.Map;
+import java.util.Map.Entry;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
@@ -55,7 +57,14 @@ public class MapReduceMain extends LauncherMain {
// Run a config class if given to update the job conf
runConfigClass(jobConf);
- logMasking("Map-Reduce job configuration:", new HashSet<String>(), jobConf);
+ PasswordMasker passwordMasker = new PasswordMasker();
+ // Temporary JobConf object, we mask out possible passwords before we print key-value pairs
+ JobConf maskedJobConf = new JobConf(false);
+ for (Entry<String, String> entry : jobConf) {
+ maskedJobConf.set(entry.getKey(), passwordMasker.maskPasswordsIfNecessary(entry.getValue()));
+ }
+
+ logMasking("Map-Reduce job configuration:", new ArrayList<String>(), maskedJobConf);
File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
System.out.println("Submitting Oozie action Map-Reduce job");
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
new file mode 100644
index 0000000..eb60aac
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.collect.Maps;
+
+import javax.annotation.Nonnull;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys.
+ * <p/>
+ * Tested with {@see System#getProperties()} and {@see System#getenv()}.
+ */
+public class PasswordMasker {
+
+ /**
+ * The mask that is applied to recognized passwords.
+ **/
+ private static final String PASSWORD_MASK = "*****";
+
+ /**
+ * A key is considered a password key, if it contains {{pass}}, case ignored.
+ **/
+ private static final String PASSWORD_KEY = "pass";
+
+ /**
+ * Tells us whether a given string contains a password fragment. A password fragment is something that looks
+ * like {{-Djavax.net.ssl.trustStorePassword=password}} or {{HADOOP_CREDSTORE_PASSWORD=pwd123}}
+ *
+ **/
+ private static final String PASSWORD_CONTAINING_REGEX =
+ "(.*)([\\w[.\\w]*]*(?i)" + PASSWORD_KEY + "[\\w]*=)([\\w]+)(.*)";
+
+ private static final Pattern PASSWORD_CONTAINING_PATTERN = Pattern
+ .compile(PASSWORD_CONTAINING_REGEX);
+
+ /**
+ * Extracts a password fragment from a given string.
+ * <p/>
+ * {@see java.util.Matcher#find()}
+ **/
+ private static final String PASSWORD_EXTRACTING_REGEX =
+ "([\\w[.\\w]*]*(?i)pass[\\w]*=)([\\w]+)";
+
+ private static final Pattern PASSWORD_EXTRACTING_PATTERN = Pattern
+ .compile(PASSWORD_EXTRACTING_REGEX);
+
+ /**
+ * Returns a map where keys are masked if they are considered a password.
+ * There are two cases when passwords are masked:
+ * 1. The key contains the string "pass". In this case, the entire value is considered a password and replaced completely with
+ * a masking string.
+ * 2. The value matches a regular expression. Strings like "HADOOP_CREDSTORE_PASSWORD=pwd123" or
+ * "-Djavax.net.ssl.trustStorePassword=password" are considered password definition strings and the text after the equal sign
+ * is replaced with a masking string.
+ *
+ * @param unmasked key-value map
+ * @return A new map where values are changed based on the replace algorithm described above
+ */
+ public Map<String, String> mask(Map<String, String> unmasked) {
+ return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() {
+ @Override
+ public String transformEntry(@Nonnull String key, @Nonnull String value) {
+ checkNotNull(key, "key has to be set");
+ checkNotNull(value, "value has to be set");
+
+ if (isPasswordKey(key)) {
+ return PASSWORD_MASK;
+ }
+
+ return maskPasswordsIfNecessary(value);
+ }
+ });
+ }
+
+ /**
+ * Masks passwords inside a string. A substring is subject to password masking if it looks like
+ * "HADOOP_CREDSTORE_PASSWORD=pwd123" or "-Djavax.net.ssl.trustStorePassword=password". The text after the equal sign is
+ * replaced with a masking string.
+ *
+ * @param unmasked String which might contain passwords
+ * @return The same string where passwords are replaced with a masking string. If there is no password inside, the original
+ * string is returned.
+ */
+ public String maskPasswordsIfNecessary(String unmasked) {
+ if (containsPasswordFragment(unmasked)) {
+ return maskPasswordFragments(unmasked);
+ } else {
+ return unmasked;
+ }
+ }
+
+ private boolean isPasswordKey(String key) {
+ return key.toLowerCase().contains(PASSWORD_KEY);
+ }
+
+ private boolean containsPasswordFragment(String maybePasswordFragments) {
+ return PASSWORD_CONTAINING_PATTERN
+ .matcher(maybePasswordFragments)
+ .matches();
+ }
+
+ private String maskPasswordFragments(String maybePasswordFragments) {
+ StringBuilder maskedBuilder = new StringBuilder();
+ Matcher passwordFragmentsMatcher = PASSWORD_EXTRACTING_PATTERN
+ .matcher(maybePasswordFragments);
+
+ int start = 0, end;
+ while (passwordFragmentsMatcher.find()) {
+ end = passwordFragmentsMatcher.start();
+
+ maskedBuilder.append(maybePasswordFragments.substring(start, end));
+ maskedBuilder.append(passwordFragmentsMatcher.group(1));
+ maskedBuilder.append(PASSWORD_MASK);
+
+ start = passwordFragmentsMatcher.end();
+ }
+
+ maskedBuilder.append(maybePasswordFragments.substring(start));
+
+ return maskedBuilder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
new file mode 100644
index 0000000..08e55e1
--- /dev/null
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestPasswordMasker {
+ private PasswordMasker passwordMasker;
+
+ @Before
+ public void setup() {
+ passwordMasker = new PasswordMasker();
+ }
+
+ @Test
+ public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception {
+ Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-system-properties.json"));
+
+ assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword");
+ assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass");
+ }
+
+ @Test
+ public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception {
+ Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-os-env.json"));
+
+ assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD");
+ assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD");
+ assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD");
+
+ assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass=");
+ assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword=");
+
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m");
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore");
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path=");
+ }
+
+ @Test
+ public void testMaskNothing() {
+ assertEquals("abcd", passwordMasker.maskPasswordsIfNecessary("abcd"));
+ assertEquals("abcd abcd", passwordMasker.maskPasswordsIfNecessary("abcd abcd"));
+ assertEquals("-Djava.net.pasX=pwd1", passwordMasker.maskPasswordsIfNecessary("-Djava.net.pasX=pwd1"));
+ }
+
+ @Test
+ public void testMaskJavaSystemProp() {
+ assertEquals("-Djava.sysprop.password=*****", passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123"));
+ }
+
+ @Test
+ public void testMaskJavaSystemPropWithWhiteSpaces() {
+ assertEquals(" -Djava.sysprop.password=***** ",
+ passwordMasker.maskPasswordsIfNecessary(" -Djava.sysprop.password=pwd123 "));
+ }
+
+ @Test
+ public void testMaskTwoJavaSystemProps() {
+ assertEquals("-Djava.sysprop.password=***** -Djava.another.password=*****",
+ passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123 -Djava.another.password=pwd456"));
+ }
+
+ @Test
+ public void testMaskEnvironmentVariable() {
+ assertEquals("DUMMY_PASSWORD=*****", passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy"));
+ }
+
+ @Test
+ public void testMaskTwoEnvironmentVariables() {
+ assertEquals("DUMMY_PASSWORD=*****:ANOTHER_PASSWORD=*****",
+ passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy:ANOTHER_PASSWORD=pwd123"));
+ }
+
+ @Test
+ public void testMaskRandomMatchingStuff() {
+ assertEquals("aa -Djava.sysprop.password=***** bb DUMMY_PASSWORD=***** cc",
+ passwordMasker.maskPasswordsIfNecessary("aa -Djava.sysprop.password=1234 bb DUMMY_PASSWORD=dummy cc"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> jsonToMap(String jsonPath) throws IOException {
+ return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class);
+ }
+
+ private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) {
+ assertEquals(String.format("Value of key '%s' should be masked.", passwordKey),
+ "*****",
+ mapContainingMaskedPassword.get(passwordKey));
+ }
+
+ private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey,
+ String passwordFragmentKey) {
+ assertEquals(
+ String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.",
+ passwordKey,
+ passwordFragmentKey),
+ "*****",
+ getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey));
+ }
+
+ private String getFragmentValue(String base, String fragmentKey) {
+ for (String fragment : base.split(" ")) {
+ if (fragment.startsWith(fragmentKey)) {
+ return fragment.substring(fragmentKey.length());
+ }
+ }
+
+ return null;
+ }
+
+ private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) {
+ assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment),
+ masked.get(key).contains(valueFragment));
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/resources/instrumentation-os-env.json
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/resources/instrumentation-os-env.json b/sharelib/oozie/src/test/resources/instrumentation-os-env.json
new file mode 100644
index 0000000..e85cd8d
--- /dev/null
+++ b/sharelib/oozie/src/test/resources/instrumentation-os-env.json
@@ -0,0 +1,47 @@
+{
+ "HADOOP_CREDSTORE_PASSWORD": "password",
+ "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password",
+ "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password",
+ "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
+ "HISTCONTROL": "ignoreboth",
+ "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
+ "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid",
+ "MC_SID": "8597",
+ "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local",
+ "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local",
+ "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home",
+ "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out",
+ "TERM": "xterm-256color",
+ "LANG": "en_US.UTF-8",
+ "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "OOZIE_CONFIG_FILE": "oozie-site.xml",
+ "LOGNAME": "forsage",
+ "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "XPC_SERVICE_NAME": "0",
+ "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "TERM_PROGRAM_VERSION": "361.1",
+ "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap",
+ "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java",
+ "SHELL": "/bin/bash",
+ "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
+ "TERM_PROGRAM": "Apple_Terminal",
+ "OOZIE_ADMIN_PORT": "11001",
+ "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users
/forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=",
+ "USER": "forsage",
+ "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
+ "OOZIE_LOG4J_RELOAD": "10",
+ "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/",
+ "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners",
+ "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage",
+ "XPC_FLAGS": "0x0",
+ "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie",
+ "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C",
+ "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore",
+ "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0",
+ "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render",
+ "OOZIE_HTTP_PORT": "11000",
+ "OOZIE_HTTPS_PORT": "11443",
+ "SHLVL": "3",
+ "HOME": "/Users/forsage",
+ "OOZIE_LOG4J_FILE": "oozie-log4j.properties"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/resources/instrumentation-system-properties.json b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
new file mode 100644
index 0000000..61430d2
--- /dev/null
+++ b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
@@ -0,0 +1,88 @@
+{
+ "javax.net.ssl.trustStorePassword": "password",
+ "oozie.https.keystore.pass": "password",
+ "gopherProxySet": "false",
+ "awt.toolkit": "sun.lwawt.macosx.LWCToolkit",
+ "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie",
+ "file.encoding.pkg": "sun.io",
+ "java.specification.version": "1.8",
+ "sun.cpu.isalist": "",
+ "sun.jnu.encoding": "UTF-8",
+ "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar",
+ "java.vm.vendor": "Oracle Corporation",
+ "sun.arch.data.model": "64",
+ "sun.font.fontmanager": "sun.font.CFontManager",
+ "catalina.useNaming": "true",
+ "java.vendor.url": "http://java.oracle.com/",
+ "user.timezone": "Europe/Budapest",
+ "os.name": "Mac OS X",
+ "java.vm.specification.version": "1.8",
+ "oozie.http.hostname": "Budapests-MacBook-Pro.local",
+ "oozie.instance.id": "Budapests-MacBook-Pro.local",
+ "sun.java.launcher": "SUN_STANDARD",
+ "user.country": "US",
+ "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
+ "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib",
+ "sun.java.command": "org.apache.catalina.startup.Bootstrap start",
+ "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
+ "sun.cpu.endian": "little",
+ "user.home": "/Users/forsage",
+ "user.language": "en",
+ "java.specification.vendor": "Oracle Corporation",
+ "java.naming.factory.url.pkgs": "org.apache.naming",
+ "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre",
+ "oozie.config.file": "oozie-site.xml",
+ "oozie.log4j.reload": "10",
+ "file.separator": "/",
+ "oozie.https.keystore.file": "/Users/forsage/.keystore",
+ "line.separator": "\n",
+ "java.vm.specification.vendor": "Oracle Corporation",
+ "java.specification.name": "Java Platform API Specification",
+ "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log",
+ "oozie.log4j.file": "oozie-log4j.properties",
+ "oozie.admin.port": "11001",
+ "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment",
+ "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.",
+ "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.",
+ "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes",
+ "server.loader": "",
+ "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties",
+ "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
+ "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
+ "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
+ "java.runtime.version": "1.8.0_102-b14",
+ "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory",
+ "user.name": "forsage",
+ "oozie.https.port": "11443",
+ "path.separator": ":",
+ "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar",
+ "os.version": "10.11.6",
+ "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed",
+ "java.runtime.name": "Java(TM) SE Runtime Environment",
+ "file.encoding": "UTF-8",
+ "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM",
+ "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/",
+ "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp",
+ "oozie.http.port": "11000",
+ "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "java.version": "1.8.0_102",
+ "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
+ "os.arch": "x86_64",
+ "java.vm.specification.name": "Java Virtual Machine Specification",
+ "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob",
+ "sun.os.patch.level": "unknown",
+ "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "shared.loader": "",
+ "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager",
+ "java.library.path": "",
+ "java.vendor": "Oracle Corporation",
+ "java.vm.info": "mixed mode",
+ "java.vm.version": "25.102-b14",
+ "sun.io.unicode.encoding": "UnicodeBig",
+ "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java",
+ "tomcat.util.buf.StringCache.byte.enabled": "true",
+ "java.class.version": "52.0",
+ "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index db1e197..88ac64e 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -263,8 +263,9 @@ public class SparkMain extends LauncherMain {
System.out.println("Oozie Spark action configuration");
System.out.println("=================================================================");
System.out.println();
+ PasswordMasker passwordMasker = new PasswordMasker();
for (String arg : sparkArgs) {
- System.out.println(" " + arg);
+ System.out.println(" " + passwordMasker.maskPasswordsIfNecessary(arg));
}
System.out.println();
try {