You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by di...@apache.org on 2022/11/29 15:35:19 UTC
[oozie] branch master updated: OOZIE-3674 Add a --insecure like parameter to Oozie client so it can ignore certificate errors (jmakai via dionusos)
This is an automated email from the ASF dual-hosted git repository.
dionusos pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 0f81e0f2d OOZIE-3674 Add a --insecure like parameter to Oozie client so it can ignore certificate errors (jmakai via dionusos)
0f81e0f2d is described below
commit 0f81e0f2db0d9749f58c71f602dbc3d473c1a09a
Author: Denes Bodo <di...@apache.org>
AuthorDate: Tue Nov 29 16:32:47 2022 +0100
OOZIE-3674 Add a --insecure like parameter to Oozie client so it can ignore certificate errors (jmakai via dionusos)
---
client/pom.xml | 8 +
client/spotbugs-filter.xml | 25 ++
.../main/java/org/apache/oozie/cli/OozieCLI.java | 45 ++-
.../org/apache/oozie/client/AuthOozieClient.java | 33 +-
.../oozie/client/InsecureConnectionHelper.java | 82 +++++
.../java/org/apache/oozie/client/OozieClient.java | 2 +-
.../oozie/client/TestInsecureConnection.java | 343 +++++++++++++++++++++
docs/src/site/markdown/DG_CommandLineTool.md | 59 +++-
release-log.txt | 1 +
9 files changed, 583 insertions(+), 15 deletions(-)
diff --git a/client/pom.xml b/client/pom.xml
index c5bb693ca..dd2fa9a0e 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -119,6 +119,14 @@
</resource>
</resources>
<plugins>
+ <!-- spotbugs plugin. Execute 'mvn verify' and look for target/spotbugs/spotbugsXml.html under each module -->
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/spotbugs-filter.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
diff --git a/client/spotbugs-filter.xml b/client/spotbugs-filter.xml
new file mode 100644
index 000000000..81634d660
--- /dev/null
+++ b/client/spotbugs-filter.xml
@@ -0,0 +1,25 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!-- InsecureConnectionHelper provides possibility to configure insecure connection so the `WEAK_TRUST_MANAGER`
+ SpotBugs pattern is invalid there.-->
+ <Match>
+ <Class name="org.apache.oozie.client.InsecureConnectionHelper$1"/>
+ <Bug pattern="WEAK_TRUST_MANAGER" />
+ </Match>
+</FindBugsFilter>
\ No newline at end of file
diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
index c22c87808..d72864d14 100644
--- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
+++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
@@ -131,6 +131,7 @@ public class OozieCLI {
public static final String VALIDATE_JAR_OPTION = "validatejar";
public static final String SUBMIT_JAR_OPTION = "submitjar";
public static final String RUN_JAR_OPTION = "runjar";
+ public static final String INSECURE_ACCESS_OPTION = "insecure";
public static final String ACTION_OPTION = "action";
public static final String DEFINITION_OPTION = "definition";
@@ -299,10 +300,14 @@ public class OozieCLI {
sharelib.setOptionalArg(true);
Option purge = new Option(PURGE_OPTION, true, "purge old oozie workflow, coordinator and bundle records from DB " +
"(parameter unit: day)");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
Options adminOptions = new Options();
adminOptions.addOption(oozie);
adminOptions.addOption(doAs);
+ adminOptions.addOption(insecureAccess);
OptionGroup group = new OptionGroup();
group.addOption(system_mode);
group.addOption(status);
@@ -416,6 +421,9 @@ public class OozieCLI {
Option workflowActionRetries = new Option(WORKFLOW_ACTIONS_RETRIES, true,
"Get information of the retry attempts for a given workflow action");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
OptionGroup actions = new OptionGroup();
actions.addOption(submit);
@@ -470,6 +478,7 @@ public class OozieCLI {
jobOptions.addOption(logFilter);
jobOptions.addOption(timeout);
jobOptions.addOption(interval);
+ jobOptions.addOption(insecureAccess);
addAuthOptions(jobOptions);
jobOptions.addOption(showdiff);
@@ -513,6 +522,10 @@ public class OozieCLI {
"coordinators and actionstatus can be multiple comma separated values. " +
"Bundle and coordinators can be id(s) or appName(s) of those jobs. " +
"Specifying bundle is mandatory, other params are optional");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
+
start.setType(Integer.class);
len.setType(Integer.class);
Options jobsOptions = new Options();
@@ -530,6 +543,7 @@ public class OozieCLI {
jobsOptions.addOption(jobtype);
jobsOptions.addOption(verbose);
jobsOptions.addOption(bulkMonitor);
+ jobsOptions.addOption(insecureAccess);
addAuthOptions(jobsOptions);
return jobsOptions;
}
@@ -544,6 +558,10 @@ public class OozieCLI {
Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')");
Option len = new Option(LEN_OPTION, true, "number of results (default '100', max '1000')");
Option filter = new Option(FILTER_OPTION, true, "filter of SLA events. e.g., jobid=<J>\\;appname=<A>");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
+
start.setType(Integer.class);
len.setType(Integer.class);
Options slaOptions = new Options();
@@ -551,6 +569,7 @@ public class OozieCLI {
slaOptions.addOption(len);
slaOptions.addOption(filter);
slaOptions.addOption(oozie);
+ slaOptions.addOption(insecureAccess);
addAuthOptions(slaOptions);
return slaOptions;
}
@@ -562,8 +581,12 @@ public class OozieCLI {
*/
protected Options createValidateOptions() {
Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
Options validateOption = new Options();
validateOption.addOption(oozie);
+ validateOption.addOption(insecureAccess);
addAuthOptions(validateOption);
return validateOption;
}
@@ -583,6 +606,10 @@ public class OozieCLI {
Option params = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
"set parameters for script").create("P");
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
+
Options Options = new Options();
Options.addOption(oozie);
Options.addOption(doAs);
@@ -590,6 +617,7 @@ public class OozieCLI {
Options.addOption(property);
Options.addOption(params);
Options.addOption(file);
+ Options.addOption(insecureAccess);
addAuthOptions(Options);
return Options;
}
@@ -607,12 +635,17 @@ public class OozieCLI {
Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
"set/override value for given property").create("D");
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
+
Options Options = new Options();
Options.addOption(oozie);
Options.addOption(doAs);
Options.addOption(config);
Options.addOption(property);
Options.addOption(command);
+ Options.addOption(insecureAccess);
addAuthOptions(Options);
return Options;
}
@@ -639,11 +672,16 @@ public class OozieCLI {
Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
"set/override value for given property").create("D");
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
+ Option insecureAccess = new Option(INSECURE_ACCESS_OPTION, false, "This option will allow SSL connections " +
+ "even though there's a problem with the certificate. The connection will still be encrypted, " +
+ "but Oozie client won't validate the server certificate.");
+
Options mrOptions = new Options();
mrOptions.addOption(oozie);
mrOptions.addOption(doAs);
mrOptions.addOption(config);
mrOptions.addOption(property);
+ mrOptions.addOption(insecureAccess);
addAuthOptions(mrOptions);
return mrOptions;
}
@@ -991,13 +1029,18 @@ public class OozieCLI {
* @throws OozieCLIException thrown if the XOozieClient could not be configured.
*/
protected XOozieClient createXOozieClient(CommandLine commandLine) throws OozieCLIException {
- XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine));
+ XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine),
+ isInsecureConnEnabled(commandLine));
addHeader(wc, commandLine);
setDebugMode(wc,commandLine.hasOption(DEBUG_OPTION));
setRetryCount(wc);
return wc;
}
+ private boolean isInsecureConnEnabled(CommandLine commandLine) {
+ return commandLine.hasOption(INSECURE_ACCESS_OPTION);
+ }
+
protected void setDebugMode(OozieClient wc, boolean debugOpt) {
String debug = System.getenv(ENV_OOZIE_DEBUG);
diff --git a/client/src/main/java/org/apache/oozie/client/AuthOozieClient.java b/client/src/main/java/org/apache/oozie/client/AuthOozieClient.java
index cad9cf569..aa65f63bb 100644
--- a/client/src/main/java/org/apache/oozie/client/AuthOozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/AuthOozieClient.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -36,6 +36,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Base64;
+import javax.net.ssl.HttpsURLConnection;
+
import com.google.common.annotations.VisibleForTesting;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -69,6 +71,7 @@ public class AuthOozieClient extends XOozieClient {
}
private String authOption = null;
+ private boolean isInsecureConnEnabled = false;
/**
* authTokenCacheFile defines the location of the authentication token cache file.
@@ -94,8 +97,21 @@ public class AuthOozieClient extends XOozieClient {
*/
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "FilenameUtils is used to filter user input. JDK8+ is used.")
public AuthOozieClient(String oozieUrl, String authOption) {
+ this(oozieUrl, authOption, false);
+ }
+
+ /**
+ * Create an instance of the AuthOozieClient.
+ *
+ * @param oozieUrl the Oozie URL
+ * @param authOption the auth option
+ * @param isInsecureConnEnabled option for handling potential certificate errors
+ */
+ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "FilenameUtils is used to filter user input. JDK8+ is used.")
+ public AuthOozieClient(String oozieUrl, String authOption, boolean isInsecureConnEnabled) {
super(oozieUrl);
this.authOption = authOption;
+ this.isInsecureConnEnabled = isInsecureConnEnabled;
String filename = getAuthCacheFileName(oozieUrl);
// just to filter user input
authTokenCacheFile = new File(System.getProperty("user.home"), FilenameUtils.getName(filename));
@@ -114,6 +130,18 @@ public class AuthOozieClient extends XOozieClient {
return filename;
}
+ private void configureConnection(HttpURLConnection httpURLConnection) {
+ if (isInsecureConnEnabled && httpURLConnection instanceof HttpsURLConnection) {
+ InsecureConnectionHelper.configureInsecureConnection((HttpsURLConnection) httpURLConnection);
+ }
+ }
+
+ private void configureAuthenticator(Authenticator authenticator) {
+ if (isInsecureConnEnabled) {
+ InsecureConnectionHelper.configureInsecureAuthenticator(authenticator);
+ }
+ }
+
/**
* Create an authenticated connection to the Oozie server.
* <p>
@@ -162,6 +190,7 @@ public class AuthOozieClient extends XOozieClient {
// If we have a token, double check with the Server to make sure it hasn't expired yet
if (currentToken.isSet()) {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ configureConnection(conn);
conn.setRequestMethod("OPTIONS");
AuthenticatedURL.injectToken(conn, currentToken);
if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED
@@ -193,6 +222,7 @@ public class AuthOozieClient extends XOozieClient {
// If we didn't have a token, or it had expired, let's get a new one from the Server using the configured Authenticator
if (!currentToken.isSet()) {
Authenticator authenticator = getAuthenticator();
+ configureAuthenticator(authenticator);
try {
authenticator.authenticate(url, currentToken);
}
@@ -216,6 +246,7 @@ public class AuthOozieClient extends XOozieClient {
// Now create a connection using the token and return it to the caller
HttpURLConnection conn = super.createConnection(url, method);
+ configureConnection(conn);
AuthenticatedURL.injectToken(conn, currentToken);
return conn;
}
diff --git a/client/src/main/java/org/apache/oozie/client/InsecureConnectionHelper.java b/client/src/main/java/org/apache/oozie/client/InsecureConnectionHelper.java
new file mode 100644
index 000000000..505711e67
--- /dev/null
+++ b/client/src/main/java/org/apache/oozie/client/InsecureConnectionHelper.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.client;
+
+import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.cert.X509Certificate;
+
+public final class InsecureConnectionHelper {
+ private static final ConnectionConfigurator INSECURE_CONN_CONFIGURATOR = httpURLConnection -> {
+ try {
+ if (httpURLConnection instanceof HttpsURLConnection) {
+ configureInsecureConnection((HttpsURLConnection) httpURLConnection);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Configuring HttpsUrlConnection failed", e);
+ }
+ return httpURLConnection;
+ };
+
+ private static final HostnameVerifier INSECURE_HOSTNAME_VERIFIER = (s, session) -> true;
+
+ private static final TrustManager[] TRUST_ALL_CERTS_MANAGER = new TrustManager[] {
+ new X509TrustManager() {
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ public void checkClientTrusted(
+ X509Certificate[] certs, String authType) {
+ }
+ public void checkServerTrusted(
+ X509Certificate[] certs, String authType) {
+ }
+ }
+ };
+
+ private static final SSLSocketFactory INSECURE_SSL_SOCKET_FACTORY;
+
+ static {
+ try {
+ SSLContext sslcontext = SSLContext.getInstance("TLS");
+ sslcontext.init(null, TRUST_ALL_CERTS_MANAGER, null);
+ INSECURE_SSL_SOCKET_FACTORY = sslcontext.getSocketFactory();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to initialize the insecure SSL socket factory", e);
+ }
+ }
+
+ private InsecureConnectionHelper() {} // no instances
+
+ public static void configureInsecureConnection(HttpsURLConnection httpsURLConnection) {
+ httpsURLConnection.setHostnameVerifier(INSECURE_HOSTNAME_VERIFIER);
+ httpsURLConnection.setSSLSocketFactory(INSECURE_SSL_SOCKET_FACTORY);
+ }
+
+ public static void configureInsecureAuthenticator(Authenticator authenticator) {
+ authenticator.setConnectionConfigurator(INSECURE_CONN_CONFIGURATOR);
+ }
+}
diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java
index 1c6966fd1..eccea8ecc 100644
--- a/client/src/main/java/org/apache/oozie/client/OozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/core/src/test/java/org/apache/oozie/client/TestInsecureConnection.java b/core/src/test/java/org/apache/oozie/client/TestInsecureConnection.java
new file mode 100644
index 000000000..c6a9207e3
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/client/TestInsecureConnection.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.client;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.oozie.cli.CLIParser;
+import org.apache.oozie.cli.OozieCLI;
+import org.apache.oozie.cli.OozieCLIException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import javax.net.ssl.HttpsURLConnection;
+import java.io.DataOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Properties;
+
+import static org.mockito.ArgumentMatchers.any;
+
+@RunWith(Parameterized.class)
+public class TestInsecureConnection {
+ private static final String OOZIE_URL = "https://foobar"; // fake Oozie https URL
+
+ private String[] oozieArgs;
+
+ private MockedStatic<InsecureConnectionHelper> insecureConnectionHelperMockedStatic;
+
+ // `createXOozieClient` is exposed in order to be able to create the `XOozieClient` which then creates the
+ // AuthOozieClient.
+ static class MyOozieCLI extends OozieCLI {
+ @Override
+ public XOozieClient createXOozieClient(CommandLine commandLine) throws OozieCLIException {
+ return super.createXOozieClient(commandLine);
+ }
+ }
+
+ MyOozieCLI myOozieCLI = new MyOozieCLI();
+ final CLIParser parser = myOozieCLI.getCLIParser();
+
+ private static final String[] adminVersionWithInsecureOption = new String[]{
+ "admin",
+ "-version",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] adminVersionWithoutInsecureOption = new String[]{
+ "admin",
+ "-version",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] jobsWithInsecureOption = new String[]{
+ "jobs",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] jobsWithoutInsecureOption = new String[]{
+ "jobs",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] jobKillWithInsecureOption = new String[]{
+ "job",
+ "-kill", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] jobKillWithoutInsecureOption = new String[]{
+ "job",
+ "-kill", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] slaFilterWithInsecureOption = new String[]{
+ "sla",
+ "-filter", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] slaFilterWithoutInsecureOption = new String[]{
+ "sla",
+ "-filter", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] pigWithInsecureOption = new String[]{
+ "pig",
+ "-config", createPropertiesFile(),
+ "-file", createPigScript(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] pigWithoutInsecureOption = new String[]{
+ "pig",
+ "-config", createPropertiesFile(),
+ "-file", createPigScript(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] hiveWithInsecureOption = new String[]{
+ "hive",
+ "-config", createPropertiesFile(),
+ "-file", createHiveScript(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] hiveWithoutInsecureOption = new String[]{
+ "hive",
+ "-config", createPropertiesFile(),
+ "-file", createHiveScript(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] sqoopWithInsecureOption = new String[]{
+ "sqoop",
+ "-config", createPropertiesFile(),
+ "-command", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] sqoopWithoutInsecureOption = new String[]{
+ "sqoop",
+ "-config", createPropertiesFile(),
+ "-command", "foobar",
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] mapreduceWithInsecureOption = new String[]{
+ "mapreduce",
+ "-config", createPropertiesFile(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", "",
+ "-insecure"
+ };
+
+ private static final String[] mapreduceWithoutInsecureOption = new String[]{
+ "mapreduce",
+ "-config", createPropertiesFile(),
+ "-oozie", OOZIE_URL,
+ "-username", "sa",
+ "-password", ""
+ };
+
+ private static final String[] validateWithInsecureOption = new String[]{
+ "validate",
+ "-username", "sa",
+ "-password", "",
+ "-oozie", OOZIE_URL,
+ "-insecure",
+ createDummyWorkflowFile()
+ };
+
+ private static final String[] validateWithoutInsecureOption = new String[]{
+ "validate",
+ "-username", "sa",
+ "-password", "",
+ "-oozie", OOZIE_URL,
+ createDummyWorkflowFile()
+ };
+
+ public TestInsecureConnection(String testName, String[] oozieArgs) throws Exception {
+ this.oozieArgs = oozieArgs;
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection testCases() {
+ return Arrays.asList(new Object[][]{
+ {"Oozie admin version with -insecure option", adminVersionWithInsecureOption},
+ {"Oozie admin version without -insecure option", adminVersionWithoutInsecureOption},
+ {"Oozie jobs with -insecure option", jobsWithInsecureOption},
+ {"Oozie jobs without -insecure option", jobsWithoutInsecureOption},
+ {"Oozie job kill with -insecure option", jobKillWithInsecureOption},
+ {"Oozie job kill without -insecure option", jobKillWithoutInsecureOption},
+ {"Oozie sla filter with -insecure option", slaFilterWithInsecureOption},
+ {"Oozie sla filter without -insecure option", slaFilterWithoutInsecureOption},
+ {"Oozie pig with -insecure option", pigWithInsecureOption},
+ {"Oozie pig without -insecure option", pigWithoutInsecureOption},
+ {"Oozie hive with -insecure option", hiveWithInsecureOption},
+ {"Oozie hive without -insecure option", hiveWithoutInsecureOption},
+ {"Oozie sqoop with -insecure option", sqoopWithInsecureOption},
+ {"Oozie sqoop without -insecure option", sqoopWithoutInsecureOption},
+ {"Oozie mapreduce with -insecure option", mapreduceWithInsecureOption},
+ {"Oozie mapreduce without -insecure option", mapreduceWithoutInsecureOption},
+ {"Oozie validate with -insecure option", validateWithInsecureOption},
+ {"Oozie validate without -insecure option", validateWithoutInsecureOption}
+ });
+ }
+
+ @Before
+ public void initTest() {
+ insecureConnectionHelperMockedStatic = Mockito.mockStatic(InsecureConnectionHelper.class);
+ }
+
+ @After
+ public void teardownTest() {
+ insecureConnectionHelperMockedStatic.close();
+ }
+
+ @Test
+ public void test() throws ParseException, OozieCLIException, IOException, OozieClientException {
+ // creating the Oozie client based on the parameterized Oozie arguments
+ XOozieClient createdOozieClient = myOozieCLI.createXOozieClient(parser.parse(oozieArgs).getCommandLine());
+
+ // invoking the `createConnection` which is responsible for configuring the insecure connection in case
+ // -insecure argument is being provided
+ createdOozieClient.createConnection(new URL(OOZIE_URL), "POST");
+
+ if (Arrays.asList(oozieArgs).contains("-insecure")) {
+ insecureConnectionHelperMockedStatic.verify(() ->
+ InsecureConnectionHelper.configureInsecureAuthenticator(any(Authenticator.class)));
+ insecureConnectionHelperMockedStatic.verify(() ->
+ InsecureConnectionHelper.configureInsecureConnection(any(HttpsURLConnection.class)));
+ } else {
+ insecureConnectionHelperMockedStatic.verifyNoInteractions();
+ }
+ }
+
+ private static String createPropertiesFile() {
+ String path = "tmp.properties";
+ Properties props = new Properties();
+ props.setProperty(OozieClient.USER_NAME, "sa");
+ props.setProperty(XOozieClient.NN, "localhost:8020");
+ props.setProperty(XOozieClient.RM, "localhost:8032");
+ props.setProperty("oozie.libpath", "/tmp");
+ props.setProperty("mapred.input.dir", "/tmp");
+ props.setProperty("mapred.output.dir", "/tmp");
+ props.setProperty("mapred.mapper.class", "foobar");
+ props.setProperty("mapred.reducer.class", "foobar");
+ props.setProperty("a", "A");
+
+ try (OutputStream os = new FileOutputStream(path)) {
+ props.store(os, "");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return path;
+ }
+
+ private static String createPigScript() {
+ String path = "pig.script";
+ String pigScript = "A = load '/user/data' using PigStorage(:);\n" +
+ "B = foreach A generate $0" +
+ "dumb B;";
+
+ try (DataOutputStream dos = new DataOutputStream(new FileOutputStream(path))) {
+ dos.writeBytes(pigScript);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return path;
+ }
+
+ private static String createHiveScript() {
+ String path = "hive.script";
+ String hiveScript = "show tables;";
+
+ try (DataOutputStream dos = new DataOutputStream(new FileOutputStream(path))) {
+ dos.writeBytes(hiveScript);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return path;
+ }
+
+ private static String createDummyWorkflowFile() {
+ String path = "workflow.xml";
+
+ try (DataOutputStream dos = new DataOutputStream(new FileOutputStream(path))) {
+ dos.writeBytes("");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return path;
+ }
+}
\ No newline at end of file
diff --git a/docs/src/site/markdown/DG_CommandLineTool.md b/docs/src/site/markdown/DG_CommandLineTool.md
index b0373bcce..e7edf1567 100644
--- a/docs/src/site/markdown/DG_CommandLineTool.md
+++ b/docs/src/site/markdown/DG_CommandLineTool.md
@@ -73,6 +73,9 @@ oozie job <OPTIONS> : job operations
-ignore <arg> change status of a coordinator job or action to IGNORED
(-action required to ignore coord actions)
-info <arg> info of a job
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-interval <arg> polling interval in minutes (default is 5, requires -poll)
-kill <arg> kill a job (coordinator can mention -action or -date)
-len <arg> number of actions (default TOTAL ACTIONS, requires -info)
@@ -144,6 +147,9 @@ oozie jobs <OPTIONS> : jobs status
'hours' or 'minutes'. startcreatedtime, endcreatedtime: time of
format yyyy-MM-dd'T'HH:mm'Z'. Valid values for sortBy are
'createdTime' or 'lastModifiedTime'.)
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-jobtype <arg> job type ('Supported in Oozie-2.0 or later versions ONLY -
'coordinator' or 'bundle' or 'wf'(default))
-kill bulk kill operation
@@ -168,6 +174,9 @@ oozie admin <OPTIONS> : admin operations
-auth <arg> select authentication type [SIMPLE|BASIC|KERBEROS]
-configuration show Oozie system configuration
-doas <arg> doAs user, impersonates as the specified user
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-instrumentation show Oozie system instrumentation
-javasysprops show Oozie Java system properties
-metrics show Oozie system metrics
@@ -194,6 +203,9 @@ oozie admin <OPTIONS> : admin operations
```
oozie validate <OPTIONS> <ARGS> : validate a workflow, coordinator, bundle XML file
-auth <arg> select authentication type [SIMPLE|BASIC|KERBEROS]
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-oozie <arg> Oozie URL
-password <arg> password for BASIC authentication
-username <arg> username for BASIC authentication
@@ -208,6 +220,10 @@ oozie sla <OPTIONS> : sla operations (Deprecated with Oozie 4.0)
-len <arg> number of results (default '100', max '1000')
-offset <arg> start offset (default '0')
-oozie <arg> Oozie URL
+ -filter <arg> jobid=<JobID/ActionID>\;appname=<Application Name>
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-password <arg> password for BASIC authentication
-username <arg> username for BASIC authentication
```
@@ -220,8 +236,10 @@ oozie pig <OPTIONS> -X <ARGS> : submit a pig job, everything after '-X' are pass
-auth <arg> select authentication type [SIMPLE|BASIC|KERBEROS]
-config <arg> job configuration file '.properties'
-D <property=value> set/override value for given property
- -doas <arg> doAs user, impersonates as the specified user
- -file <arg> pig script
+ -file <arg> Pig script
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-oozie <arg> Oozie URL
-P <property=value> set parameters for script
-password <arg> password for BASIC authentication
@@ -238,6 +256,9 @@ oozie hive <OPTIONS> -X<ARGS> : submit a hive job, everything after '-X' are pa
-D <property=value> set/override value for given property
-doas <arg> doAs user, impersonates as the specified user
-file <arg> hive script
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-oozie <arg> Oozie URL
-P <property=value> set parameters for script
-password <arg> password for BASIC authentication
@@ -247,16 +268,18 @@ oozie hive <OPTIONS> -X<ARGS> : submit a hive job, everything after '-X' are pa
### Oozie Sqoop submit command
```
-oozie sqoop <OPTIONS> -X <ARGS> : submit a sqoop job, everything after '-X' are pass-through parameters to sqoop, any '-D'
- arguments after '-X' are put in <configuration>
- -auth <arg> select authentication type [SIMPLE|BASIC|KERBEROS]
- -command <command> sqoop command
- -config <arg> job configuration file '.properties'
- -D <property=value> set/override value for given property
- -doas <arg> doAs user, impersonates as the specified user
- -oozie <arg> Oozie URL
- -password <arg> password for BASIC authentication
- -username <arg> username for BASIC authentication
+oozie sqoop <OPTIONS> -X<ARGS> : submit a sqoop job, any '-D' arguments after '-X' are put in <configuration>
+ -auth <arg> select authentication type [SIMPLE|BASIC|KERBEROS]
+ -config <arg> job configuration file '.properties'
+ -D <property=value> set/override value for given property
+ -doas <arg> doAs user, impersonates as the specified user
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
+ -command <arg> sqoop command
+ -oozie <arg> Oozie URL
+ -password <arg> password for BASIC authentication
+ -username <arg> username for BASIC authentication
```
### Oozie info command
@@ -274,6 +297,9 @@ oozie mapreduce <OPTIONS> : submit a mapreduce job
-config <arg> job configuration file '.properties'
-D <property=value> set/override value for given property
-doas <arg> doAs user, impersonates as the specified user
+ -insecure This option will allow SSL connections even though there's a problem with the
+ certificate. The connection will still be encrypted, but Oozie client won't validate
+ the server certificate.
-oozie <arg> Oozie URL
-password <arg> password for BASIC authentication
-username <arg> username for BASIC authentication
@@ -325,6 +351,15 @@ and uses it if set.
If the option is not provided and the environment variable is not set, the `oozie` CLI will fail.
+### Insecure connection
+
+In case `-insecure` option is used then it will cause Oozie to allow certificate errors where the data is still encrypted,
+but the client does not check the certificate.
+
+If the `-insecure` option is not specified and SSL is enabled, the user needs to pass the `-Djavax.net.ssl.trustStore`
+and `-Djavax.net.ssl.trustStorePassword` system properties to the Oozie client or the certificate needs to be imported
+into the JDK's cert store otherwise Oozie client won't be able to connect to the Oozie server.
+
### Time zone
The `-timezone TIME_ZONE_ID` option in the `job` and `jobs` sub-commands allows you to specify the time zone to use in
diff --git a/release-log.txt b/release-log.txt
index a41bfc8fd..0c0120670 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3674 Add a --insecure like parameter to Oozie client so it can ignore certificate errors (jmakai via dionusos)
OOZIE-3673 Add possibility to configure custom SSL/TLS protocols when executing an email action (jmakai via dionusos)
OOZIE-3675 Upgrade Mockito from 2 to 3.11.2 (jmakai via dionusos)
OOZIE-3669 Fix purge process for bundles to prevent orphan coordinators (jmakai via dionusos)