You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2014/10/17 02:28:54 UTC
git commit: OOZIE-1914 CLI should retry on timeout
Repository: oozie
Updated Branches:
refs/heads/master e717255bf -> 6f5c709c6
OOZIE-1914 CLI should retry on timeout
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6f5c709c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6f5c709c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6f5c709c
Branch: refs/heads/master
Commit: 6f5c709c67ee95ddc836489211e52571a9c02ead
Parents: e717255
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Thu Oct 16 17:28:37 2014 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Thu Oct 16 17:28:37 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/oozie/cli/OozieCLI.java | 61 ++++++++-----
.../org/apache/oozie/client/OozieClient.java | 41 +++++++--
.../client/retry/ConnectionRetriableClient.java | 92 ++++++++++++++++++++
.../apache/oozie/TestLocalOozieClientCoord.java | 4 +-
.../org/apache/oozie/client/TestOozieCLI.java | 75 ++++++++++++++++
docs/src/site/twiki/DG_CommandLineTool.twiki | 8 ++
release-log.txt | 1 +
7 files changed, 255 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
index 7f2fb3a..9c2d14b 100644
--- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
+++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
@@ -80,6 +80,7 @@ public class OozieCLI {
public static final String ENV_OOZIE_DEBUG = "OOZIE_DEBUG";
public static final String ENV_OOZIE_TIME_ZONE = "OOZIE_TIMEZONE";
public static final String ENV_OOZIE_AUTH = "OOZIE_AUTH";
+ public static final String OOZIE_RETRY_COUNT = "oozie.connection.retry.count";
public static final String WS_HEADER_PREFIX = "header:";
public static final String HELP_CMD = "help";
@@ -169,6 +170,8 @@ public class OozieCLI {
private static final String RULER;
private static final int LINE_WIDTH = 132;
+ private static final int RETRY_COUNT = 4;
+
private boolean used;
private static final String INSTANCE_SEPARATOR = "#";
@@ -547,24 +550,7 @@ public class OozieCLI {
throw new IllegalStateException("CLI instance already used");
}
used = true;
-
- final CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp());
- parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false);
- parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false);
- parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false);
- parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false);
- parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false);
- parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true);
- parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false);
- parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' "
- + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true);
- parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' "
- + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true);
- parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " +
- "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true);
- parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false);
- parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false);
-
+ final CLIParser parser = getCLIParser();
try {
final CLIParser.Command command = parser.parse(args);
@@ -582,7 +568,6 @@ public class OozieCLI {
else {
processCommand(parser, command);
}
-
return 0;
}
catch (OozieCLIException ex) {
@@ -602,7 +587,28 @@ public class OozieCLI {
}
}
- private void processCommand(CLIParser parser, CLIParser.Command command) throws Exception {
+ @VisibleForTesting
+ public CLIParser getCLIParser(){
+ CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp());
+ parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false);
+ parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false);
+ parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false);
+ parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false);
+ parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false);
+ parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true);
+ parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false);
+ parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' "
+ + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true);
+ parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' "
+ + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true);
+ parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " +
+ "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true);
+ parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false);
+ parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false);
+ return parser;
+ }
+
+ public void processCommand(CLIParser parser, CLIParser.Command command) throws Exception {
if (command.getName().equals(HELP_CMD)) {
parser.showHelp(command.getCommandLine());
}
@@ -848,6 +854,7 @@ public class OozieCLI {
XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine));
addHeader(wc);
setDebugMode(wc,commandLine.hasOption(DEBUG_OPTION));
+ setRetryCount(wc);
return wc;
}
@@ -870,6 +877,20 @@ public class OozieCLI {
}
}
+ protected void setRetryCount(OozieClient wc) {
+ String retryCount = System.getProperty(OOZIE_RETRY_COUNT);
+ if (retryCount != null && !retryCount.isEmpty()) {
+ try {
+ int retry = Integer.parseInt(retryCount.trim());
+ wc.setRetryCount(retry);
+ }
+ catch (Exception ex) {
+ System.err.println("Unable to parse the retry settings. May be not an integer [" + retryCount + "]");
+ ex.printStackTrace();
+ }
+ }
+ }
+
private static String JOB_ID_PREFIX = "job: ";
private void jobCommand(CommandLine commandLine) throws IOException, OozieCLIException {
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/client/src/main/java/org/apache/oozie/client/OozieClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java
index 5bbac4a..5e53a18 100644
--- a/client/src/main/java/org/apache/oozie/client/OozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -22,6 +22,7 @@ import org.apache.oozie.BuildInfo;
import org.apache.oozie.client.rest.JsonTags;
import org.apache.oozie.client.rest.JsonToBean;
import org.apache.oozie.client.rest.RestConstants;
+import org.apache.oozie.client.retry.ConnectionRetriableClient;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@@ -189,6 +190,9 @@ public class OozieClient {
*/
public int debugMode = 0;
+ private int retryCount = 4;
+
+
private String baseUrl;
private String protocolUrl;
private boolean validatedVersion = false;
@@ -276,6 +280,15 @@ public class OozieClient {
this.debugMode = debugMode;
}
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+
+ public void setRetryCount(int retryCount) {
+ this.retryCount = retryCount;
+ }
+
private String getBaseURLForVersion(long protocolVersion) throws OozieClientException {
try {
if (supportedVersions == null) {
@@ -343,8 +356,10 @@ public class OozieClient {
private JSONArray getSupportedProtocolVersions() throws IOException, OozieClientException {
JSONArray versions = null;
- URL url = new URL(baseUrl + RestConstants.VERSIONS);
- HttpURLConnection conn = createConnection(url, "GET");
+ final URL url = new URL(baseUrl + RestConstants.VERSIONS);
+
+ HttpURLConnection conn = createRetryableConnection(url, "GET");
+
if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
versions = (JSONArray) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
}
@@ -453,6 +468,24 @@ public class OozieClient {
}
return true;
}
+ /**
+ * Create retryable http connection to oozie server.
+ *
+ * @param url
+ * @param method
+ * @return connection
+ * @throws IOException
+ * @throws OozieClientException
+ */
+ protected HttpURLConnection createRetryableConnection(final URL url, final String method) throws IOException{
+ return (HttpURLConnection) new ConnectionRetriableClient(getRetryCount()) {
+ @Override
+ public Object doExecute(URL url, String method) throws IOException, OozieClientException {
+ HttpURLConnection conn = createConnection(url, method);
+ return conn;
+ }
+ }.execute(url, method);
+ }
/**
* Create http connection to oozie server.
@@ -501,8 +534,7 @@ public class OozieClient {
if (getDebugMode() > 0) {
System.out.println(method + " " + url);
}
- HttpURLConnection conn = createConnection(url, method);
- return call(conn);
+ return call(createRetryableConnection(url, method));
}
else {
System.out.println("Option not supported in target server. Supported only on Oozie-2.0 or greater."
@@ -513,7 +545,6 @@ public class OozieClient {
catch (IOException ex) {
throw new OozieClientException(OozieClientException.IO_ERROR, ex);
}
-
}
protected abstract T call(HttpURLConnection conn) throws IOException, OozieClientException;
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java b/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java
new file mode 100644
index 0000000..3371036
--- /dev/null
+++ b/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.client.retry;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.URL;
+
+/**
+ * HTTP connection retryable class. It retries =oozie.connection.retry.count= times for ConnectException. For
+ * SocketException all create connection are retried except =PUT= and =POST=.
+ */
+public abstract class ConnectionRetriableClient {
+ private final int retryCount;
+
+ public ConnectionRetriableClient(int retryCount) {
+ this.retryCount = retryCount;
+ }
+
+ public Object execute(URL url, String method) throws IOException {
+ int numTries = 0;
+ boolean stopRetry = false;
+ Exception cliException = null;
+
+ while (numTries < retryCount && !stopRetry) {
+ try {
+ return doExecute(url, method);
+ }
+ catch (ConnectException e) {
+ sleep(e, numTries++);
+ cliException = e;
+ }
+ catch (SocketException e) {
+ if (method.equals("POST") || method.equals("PUT")) {
+ stopRetry = true;
+ }
+ else {
+ sleep(e, numTries++);
+ }
+ cliException = e;
+ }
+ catch (Exception e) {
+ stopRetry = true;
+ cliException = e;
+ numTries++;
+ // No retry for other exceptions
+ }
+ }
+ throw new IOException("Error while connecting Oozie server. No of retries = " + numTries + ". Exception = "
+ + cliException.getMessage(), cliException);
+ }
+
+ private void sleep(Exception e, int numTries) {
+ try {
+ long wait = getWaitTimeExp(numTries);
+ System.err.println("Connection exception has occurred [ " + e.getClass().getName() + " " + e.getMessage()
+ + " ]. Trying after " + wait / 1000 + " sec." + " Retry count = " + (numTries + 1));
+ Thread.sleep(wait);
+ }
+ catch (InterruptedException e1) {
+ // Ignore InterruptedException
+ }
+ }
+
+ /*
+ * Returns the next wait interval, in milliseconds, using an exponential backoff algorithm.
+ */
+ private long getWaitTimeExp(int retryCount) {
+ long waitTime = ((long) Math.pow(2, retryCount) * 1000L);
+ return waitTime;
+ }
+
+ public abstract Object doExecute(URL url, String method) throws Exception;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
index cdb1524..4decd52 100644
--- a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
+++ b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
@@ -68,12 +68,12 @@ public class TestLocalOozieClientCoord extends XDataTestCase {
assertEquals("localoozie", client.getOozieUrl());
}
- public void testGetProtocolUrl() throws OozieClientException {
+ public void testGetProtocolUrl() throws IOException, OozieClientException {
OozieClient client = LocalOozie.getCoordClient();
assertEquals("localoozie", client.getProtocolUrl());
}
- public void testValidateWSVersion() throws OozieClientException {
+ public void testValidateWSVersion() throws IOException, OozieClientException {
OozieClient client = LocalOozie.getCoordClient();
client.validateWSVersion();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
index 74e8bca..ac54b09 100644
--- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
+++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
@@ -29,6 +29,7 @@ import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.oozie.cli.CLIParser;
import org.apache.oozie.cli.OozieCLI;
import org.apache.oozie.client.rest.RestConstants;
import org.apache.oozie.servlet.DagServletTestCase;
@@ -1255,7 +1256,81 @@ public class TestOozieCLI extends DagServletTestCase {
return null;
}
});
+ }
+ public void testRetryForTimeout() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+ String oozieUrl = "http://localhost:11/oozie";
+ String[] args = new String[] { "job", "-update", "aaa", "-dryrun", "-oozie", oozieUrl, "-debug" };
+ OozieCLI cli = new OozieCLI();
+ CLIParser parser = cli.getCLIParser();
+ try {
+ final CLIParser.Command command = parser.parse(args);
+ cli.processCommand(parser, command);
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage().contains(
+ "Error while connecting Oozie server. No of retries = 4. Exception = Connection refused"));
+ }
+ return null;
+ }
+ });
+ }
+
+ public void testNoRetryForError() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+ String oozieUrl = getContextURL();
+ String[] args = new String[] { "job", "-info", "aaa", "-oozie", oozieUrl, "-debug" };
+ OozieCLI cli = new OozieCLI();
+ CLIParser parser = cli.getCLIParser();
+ try {
+ final CLIParser.Command command = parser.parse(args);
+ cli.processCommand(parser, command);
+ }
+ catch (Exception e) {
+ //Create connection will be successful, no retry
+ assertFalse(e.getMessage().contains("Error while connecting Oozie server"));
+ assertTrue(e.getMessage().contains("invalid job id [aaa]"));
+ }
+ return null;
+ }
+ });
}
+ public void testRetryWithRetryCount() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+ String oozieUrl = "http://localhost:11/oozie";
+ String[] args = new String[] { "job", "-update", "aaa", "-dryrun", "-oozie", oozieUrl, "-debug" };
+ OozieCLI cli = new OozieCLI() {
+ protected void setRetryCount(OozieClient wc) {
+ wc.setRetryCount(2);
+ }
+ public CLIParser getCLIParser(){
+ return super.getCLIParser();
+ }
+
+ };
+ CLIParser parser = cli.getCLIParser();
+ try {
+ final CLIParser.Command command = parser.parse(args);
+ cli.processCommand(parser, command);
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage().contains(
+ "Error while connecting Oozie server. No of retries = 2. Exception = Connection refused"));
+ }
+ return null;
+ }
+ });
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/docs/src/site/twiki/DG_CommandLineTool.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki
index 00ae180..b37e947 100644
--- a/docs/src/site/twiki/DG_CommandLineTool.twiki
+++ b/docs/src/site/twiki/DG_CommandLineTool.twiki
@@ -223,6 +223,14 @@ time zone, it will use GMT.
If you export <code>OOZIE_DEBUG=1</code> then the Oozie CLI will output the Web Services API details used by any commands you
execute. This is useful for debugging purposes to or see how the Oozie CLI works with the WS API.
+---+++ CLI retry
+Oozie CLI retries connection to Oozie servers for transparent high availability failover when one of the Oozie servers go down.
+=Oozie= CLI command will retry for all commands in case of ConnectException.
+In case of SocketException, all commands except =PUT= and =POST= will have retry logic.
+All job submit are POST call, examples of PUT and POST commands can be find out from [[WebServicesAPI][WebServicesAPI]].
+Retry count can be configured with system property =oozie.connection.retry.count=. Default count is 4.
+
+
---++ Job Operations
---+++ Submitting a Workflow, Coordinator or Bundle Job
http://git-wip-us.apache.org/repos/asf/oozie/blob/6f5c709c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 64751b0..b5fc252 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-1914 CLI should retry on timeout (puru)
OOZIE-1973 ConcurrentModificationException in Sharelib service (puru)
OOZIE-1728 When an ApplicationMaster restarts, it restarts the launcher job: DistCp followup (ryota)
OOZIE-2009 Requeue CoordActionInputCheck in case of permission error (ryota)