You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by su...@apache.org on 2016/09/27 23:02:48 UTC
[1/3] incubator-trafodion git commit: [TRAFODION-2245] Multiple
sqcheck and jps processes running when monitor is downed and up as dcsserver
checks if trafodion is up
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 708f58f61 -> af63cced7
[TRAFODION-2245] Multiple sqcheck and jps processes running when monitor is downed and up
as dcsserver checks if trafodion is up
A new option sqcheck -n <nid> is introduced to determine if the node is up and running
DCS server is changed to use this option when it detects the server is gone
rather than at the time of starting the processes
Also fixed the failure of hive/TEST018 due to expected file difference
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/7bc317d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/7bc317d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/7bc317d0
Branch: refs/heads/master
Commit: 7bc317d0c44cd5e0a75ce2661687a2fd3d562cc2
Parents: 3ad30f5
Author: selvaganesang <se...@esgyn.com>
Authored: Sat Sep 24 17:03:18 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Sat Sep 24 17:03:18 2016 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/sqcheck | 27 +++++-
core/sql/regress/hive/EXPECTED018 | 3 +-
.../org/trafodion/dcs/server/ServerManager.java | 96 +++++++++++---------
3 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/7bc317d0/core/sqf/sql/scripts/sqcheck
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqcheck b/core/sqf/sql/scripts/sqcheck
index c7b971a..e4566c6 100755
--- a/core/sqf/sql/scripts/sqcheck
+++ b/core/sqf/sql/scripts/sqcheck
@@ -42,13 +42,14 @@ function Usage {
echo "-h Help"
echo "-v Verbose"
echo "-q Quiet"
+ echo "-n <nid> Check if the node is up"
echo
}
function GetOpts {
- while getopts "c:i:d:fqvrjh" arg
+ while getopts "c:i:d:fqvrjn:h" arg
do
case $arg in
c)
@@ -89,6 +90,9 @@ function GetOpts {
j)
json_output=1
;;
+ n)
+ check_node=${OPTARG}
+ ;;
h)
Usage;
exit 1;
@@ -160,6 +164,21 @@ function getRestInfo {
}
+function getNodeStatus {
+ sq_tmp_node_info=`mktemp -t`
+ sqshell -c zone nid $check_node > $sq_tmp_node_info
+ sq_tmp_lnode_up=`mktemp -t`
+ grep -i 'up' $sq_tmp_node_info | awk '{ print $4 }' | grep -i 'up' > $sq_tmp_lnode_up
+ for lnodeup in `cat $sq_tmp_lnode_up`; do
+ let ++act_lnode_up_cnt
+ done
+ if ( [ $act_lnode_up_cnt '>' 0 ] ); then
+ exit 0;
+ else
+ exit 1;
+ fi
+}
+
jpscmd=$JAVA_HOME/bin/jps
tmpjps=/tmp/tmpjps-$USER.log
begin_time_seconds=$(date +%s)
@@ -169,12 +188,14 @@ declare -i sleep_duration
declare -i sqc_verbose
declare -i reset_counter
declare -i json_output
+declare -i check_node
let max_checks=2
let sleep_duration=1
let sqc_verbose=0
let reset_counter=0
let json_output=0
+let check_node=-1
# Set debug_display to 1 to eanble debugging display of values
let debug_display=0
@@ -229,10 +250,14 @@ let sq_up=0
if [[ -z $SQSCRIPTS_DIR ]]; then
SQSCRIPTS_DIR=$MY_SQROOT/sql/scripts
fi
+
SQSCRIPT_FILE="$SQSCRIPTS_DIR/gomon.cold"
STARTSSMP_FILE="$SQSCRIPTS_DIR/ssmpstart"
STARTSSCP_FILE="$SQSCRIPTS_DIR/sscpstart"
+if [ $check_node '>' -1 ]; then
+ getNodeStatus
+fi
### CONFIGURED NODEs
sq_tmp_node_info=`mktemp -t`
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/7bc317d0/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index d28f47f..0e64b5b 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -2254,8 +2254,7 @@ C_FIRST_NAME
Task: UNLOAD Status: Started
Task: EXTRACT Status: Started
-*** ERROR[8447] An error occurred during hdfs access. Error Detail: Java exception in hdfsCreate().
-java.io.IOException: No FileSystem for scheme: null
+*** ERROR[8447] An error occurred during hdfs access. Error Detail: Java exception in hdfsCreate(). java.io.IOException: No FileSystem for scheme: null
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2584)
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2591)
org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/7bc317d0/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
----------------------------------------------------------------------
diff --git a/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java b/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
index b5e65a1..c6e864e 100644
--- a/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
+++ b/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
@@ -85,6 +85,7 @@ public final class ServerManager implements Callable {
private static String sqlplanEnable;
private static int userProgPortMapToSecs;
private static int userProgPortBindToSecs;
+ private static ServerHandler[] serverHandlers;
private int maxRestartAttempts;
private int retryIntervalMillis;
private RetryCounterFactory retryCounterFactory;
@@ -116,6 +117,8 @@ public final class ServerManager implements Callable {
String registeredPath;
Stat stat = null;
boolean isRunning = false;
+ String nid;
+ String pid;
public ServerMonitor(int childInstance, String registeredPath) {
this.childInstance = childInstance;
@@ -127,14 +130,14 @@ public final class ServerManager implements Callable {
stat = zkc.exists(registeredPath, false);
if (stat != null) { // User program znode found in
// /registered...check pid
- isRunning = isPid();
+ isRunning = isPidRunning();
LOG.debug("isRunning [" + isRunning + "]");
}
return isRunning;
}
- private boolean isPid() throws Exception {
+ private boolean isPidRunning() throws Exception {
String data = Bytes.toString(zkc.getData(registeredPath, false,
stat));
Scanner scn = new Scanner(data);
@@ -142,8 +145,8 @@ public final class ServerManager implements Callable {
scn.next();// state
scn.next();// timestamp
scn.next();// dialogue Id
- scn.next();// nid
- String pid = scn.next();// pid
+ nid = scn.next();// nid
+ pid = scn.next();// pid
scn.close();
scriptContext.setHostName(hostName);
scriptContext.setScriptName(Constants.SYS_SHELL_SCRIPT_NAME);
@@ -202,16 +205,6 @@ public final class ServerManager implements Callable {
public void exec() throws Exception {
cleanupZk();
- RetryCounter retryCounter = retryCounterFactory.create();
- while (!isTrafodionRunning()) {
- LOG.error("Trafodion is not running");
- if (!retryCounter.shouldRetry()) {
- return;
- } else {
- retryCounter.sleepUntilNextRetry();
- retryCounter.useRetry();
- }
- }
LOG.info("User program exec [" + scriptContext.getCommand() + "]");
ScriptManager.getInstance().runScript(scriptContext);// This will
// block while
@@ -249,32 +242,6 @@ public final class ServerManager implements Callable {
LOG.debug(e);
}
}
-
- private boolean isTrafodionRunning() {
-
- // Check if Trafodion is up and running
- // If Trafodion is fully or partially up and operational
- // return true else return false.
- // Invoke sqcheck to check Trafodion status.
- //
- // sqcheck returns:
- // -1 - Not up ($?=255)
- // 0 - Fully up and operational
- // 1 - Partially up and operational
- // 2 - Partially up and NOT operational
-
- ScriptContext scriptContext = new ScriptContext();
- scriptContext.setHostName(hostName);
- scriptContext.setScriptName(Constants.SYS_SHELL_SCRIPT_NAME);
- String command = ("sqcheck");
- scriptContext.setCommand(command);
- ScriptManager.getInstance().runScript(scriptContext);// This will
- // block while
- // script is
- // running
- int exitCode = scriptContext.getExitCode();
- return (exitCode == 0 || exitCode == 1) ? true : false;
- }
}
class ServerHandler implements Callable<Integer> {
@@ -367,6 +334,30 @@ public final class ServerManager implements Callable {
Constants.DEFAULT_DCS_SERVER_USER_PROGRAM_RESTART_HANDLER_RETRY_INTERVAL_MILLIS);
this.retryCounterFactory = new RetryCounterFactory(
this.maxRestartAttempts, this.retryIntervalMillis);
+ serverHandlers = new ServerHandler[this.childServers];
+ }
+
+ private static boolean isTrafodionRunning(String nid) {
+
+ // Check if the given Node is up and running
+ // return true else return false.
+ // Invoke sqcheck -n <nid> to check Node status.
+ //
+ // sqcheck returns:
+ // 0 - Node up
+ // 1 - Node down
+
+ ScriptContext scriptContext = new ScriptContext();
+ scriptContext.setHostName(hostName);
+ scriptContext.setScriptName(Constants.SYS_SHELL_SCRIPT_NAME);
+ String command = ("sqcheck -n " + nid);
+ scriptContext.setCommand(command);
+ ScriptManager.getInstance().runScript(scriptContext);// This will
+ // block while
+ // script is
+ // running
+ int exitCode = scriptContext.getExitCode();
+ return (exitCode == 0) ? true : false;
}
@Override
@@ -390,7 +381,8 @@ public final class ServerManager implements Callable {
// /bin/dcs-daemon script
// which DOES NOT set childServers count.
for (int childInstance = 1; childInstance <= childServers; childInstance++) {
- completionService.submit(new ServerHandler(childInstance));
+ serverHandlers[childInstance-1] = new ServerHandler(childInstance);
+ completionService.submit(serverHandlers[childInstance-1]);
LOG.debug("Started server handler [" + instance + ":"
+ childInstance + "]");
}
@@ -403,9 +395,29 @@ public final class ServerManager implements Callable {
// finish
if (f != null) {
Integer result = f.get();
+ int childInstance = result.intValue();
+ // get the node id
+ boolean isRunning = serverHandlers[childInstance-1].serverMonitor.isPidRunning();
+ String nid = serverHandlers[childInstance-1].serverMonitor.nid;
+ String pid = serverHandlers[childInstance-1].serverMonitor.pid;
+ serverHandlers[childInstance-1] = null;
LOG.debug("Server handler [" + instance + ":" + result
+ "] finished, restarting");
- completionService.submit(new ServerHandler(result));
+ if (isRunning)
+ LOG.error("mxosrvr " + nid + "," + pid + " still running, but restarting another");
+ else
+ LOG.error("mxosrvr " + nid + "," + pid + " exited, restarting");
+ RetryCounter retryCounter = retryCounterFactory.create();
+ while (!isTrafodionRunning(nid)) {
+ if (!retryCounter.shouldRetry()) {
+ throw new IOException("Node " + nid + " is not Up");
+ } else {
+ retryCounter.sleepUntilNextRetry();
+ retryCounter.useRetry();
+ }
+ }
+ serverHandlers[childInstance-1] = new ServerHandler(childInstance);
+ completionService.submit(serverHandlers[childInstance-1]);
}
}
[3/3] incubator-trafodion git commit: Merge [TRAFODION-2245] PR-726
Multiple sqcheck and jps processes running when monitor is downed and up as
dcsserver checks if trafodion is up
Posted by su...@apache.org.
Merge [TRAFODION-2245] PR-726 Multiple sqcheck and jps processes running when monitor is downed and up as dcsserver checks if trafodion is up
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/af63cced
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/af63cced
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/af63cced
Branch: refs/heads/master
Commit: af63cced7349b2c16612fbdf3023649d98aa5759
Parents: 708f58f 810fc3b
Author: Suresh Subbiah <su...@apache.org>
Authored: Tue Sep 27 23:02:20 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Tue Sep 27 23:02:20 2016 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/sqcheck | 27 ++++-
core/sql/regress/hive/EXPECTED018 | 3 +-
.../org/trafodion/dcs/server/ServerManager.java | 115 ++++++++++++-------
3 files changed, 100 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-trafodion git commit: [TRAFODION-2245] Multiple
sqcheck and jps processes running when monitor is downed and up
Posted by su...@apache.org.
[TRAFODION-2245] Multiple sqcheck and jps processes running when monitor is downed and up
Changes done after node down/up testing. DCS Server doesn't start mxosrvr without checking trafodion is up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/810fc3b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/810fc3b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/810fc3b3
Branch: refs/heads/master
Commit: 810fc3b35a88cf6a37a75cfde1bf43f3f2365121
Parents: 7bc317d
Author: selvaganesang <se...@esgyn.com>
Authored: Tue Sep 27 03:23:08 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Tue Sep 27 03:23:08 2016 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/sqcheck | 2 +-
.../org/trafodion/dcs/server/ServerManager.java | 35 +++++++++++++++-----
2 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/810fc3b3/core/sqf/sql/scripts/sqcheck
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqcheck b/core/sqf/sql/scripts/sqcheck
index e4566c6..b5d79ae 100755
--- a/core/sqf/sql/scripts/sqcheck
+++ b/core/sqf/sql/scripts/sqcheck
@@ -175,7 +175,7 @@ function getNodeStatus {
if ( [ $act_lnode_up_cnt '>' 0 ] ); then
exit 0;
else
- exit 1;
+ exit -1;
fi
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/810fc3b3/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
----------------------------------------------------------------------
diff --git a/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java b/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
index c6e864e..3a65662 100644
--- a/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
+++ b/dcs/src/main/java/org/trafodion/dcs/server/ServerManager.java
@@ -85,10 +85,11 @@ public final class ServerManager implements Callable {
private static String sqlplanEnable;
private static int userProgPortMapToSecs;
private static int userProgPortBindToSecs;
- private static ServerHandler[] serverHandlers;
+ private ServerHandler[] serverHandlers;
private int maxRestartAttempts;
private int retryIntervalMillis;
private RetryCounterFactory retryCounterFactory;
+ private String nid = null;
class RegisteredWatcher implements Watcher {
CountDownLatch startSignal;
@@ -344,20 +345,26 @@ public final class ServerManager implements Callable {
// Invoke sqcheck -n <nid> to check Node status.
//
// sqcheck returns:
- // 0 - Node up
- // 1 - Node down
+ // -1 - Not up ($?=255) or node down
+ // 0 - Fully up and operational or node up
+ // 1 - Partially up and operational
+ // 2 - Partially up and NOT operational
ScriptContext scriptContext = new ScriptContext();
scriptContext.setHostName(hostName);
scriptContext.setScriptName(Constants.SYS_SHELL_SCRIPT_NAME);
- String command = ("sqcheck -n " + nid);
+ String command;
+ if (nid != null)
+ command = ("sqcheck -n " + nid);
+ else
+ command = ("sqcheck");
scriptContext.setCommand(command);
ScriptManager.getInstance().runScript(scriptContext);// This will
// block while
// script is
// running
int exitCode = scriptContext.getExitCode();
- return (exitCode == 0) ? true : false;
+ return (exitCode == 0 || exitCode == 1) ? true : false;
}
@Override
@@ -372,6 +379,18 @@ public final class ServerManager implements Callable {
getMaster();
featureCheck();
registerInRunning(instance);
+ RetryCounter retryCounter = retryCounterFactory.create();
+ while (!isTrafodionRunning(nid)) {
+ if (!retryCounter.shouldRetry()) {
+ if (nid != null)
+ throw new IOException("Node " + nid + " is not Up");
+ else
+ throw new IOException("Trafodion is not running");
+ } else {
+ retryCounter.sleepUntilNextRetry();
+ retryCounter.useRetry();
+ }
+ }
// When started from bin/dcs-start.sh script childServers will
// contain the
@@ -404,10 +423,10 @@ public final class ServerManager implements Callable {
LOG.debug("Server handler [" + instance + ":" + result
+ "] finished, restarting");
if (isRunning)
- LOG.error("mxosrvr " + nid + "," + pid + " still running, but restarting another");
+ LOG.info("mxosrvr " + nid + "," + pid + " still running");
else
- LOG.error("mxosrvr " + nid + "," + pid + " exited, restarting");
- RetryCounter retryCounter = retryCounterFactory.create();
+ LOG.info("mxosrvr " + nid + "," + pid + " exited, restarting");
+ retryCounter = retryCounterFactory.create();
while (!isTrafodionRunning(nid)) {
if (!retryCounter.shouldRetry()) {
throw new IOException("Node " + nid + " is not Up");