You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/05/11 20:16:26 UTC
[1/7] accumulo git commit: ACCUMULO-3793 use numctl to turn on memory
interleaving
Repository: accumulo
Updated Branches:
refs/heads/master 5e0552741 -> ce7943d4a
ACCUMULO-3793 use numctl to turn on memory interleaving
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b8214877
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b8214877
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b8214877
Branch: refs/heads/master
Commit: b82148777c023b21578d06ef451fdd70770107a5
Parents: fa22fa4
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:07:16 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:07:16 2015 -0400
----------------------------------------------------------------------
assemble/bin/config.sh | 9 +++++++++
assemble/bin/start-server.sh | 4 ++--
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b8214877/assemble/bin/config.sh
----------------------------------------------------------------------
diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh
index 7663473..9bee91b 100755
--- a/assemble/bin/config.sh
+++ b/assemble/bin/config.sh
@@ -149,6 +149,15 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then
fi
fi
+NUMA=`which numactl 2>/dev/null`
+NUMACTL_EXISTS="$?"
+NUMACTL_ARGS="--interleave=all"
+if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then
+ export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}"
+else
+ export NUMA_CMD=""
+fi
+
SSH='ssh -qnf -o ConnectTimeout=2'
export HADOOP_HOME=$HADOOP_PREFIX
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b8214877/assemble/bin/start-server.sh
----------------------------------------------------------------------
diff --git a/assemble/bin/start-server.sh b/assemble/bin/start-server.sh
index aa1375a..daa8495 100755
--- a/assemble/bin/start-server.sh
+++ b/assemble/bin/start-server.sh
@@ -75,10 +75,10 @@ fi
if [ -z "$PID" ]; then
echo "Starting $LONGNAME on $HOST"
if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then
- ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err &
+ ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err &
MAX_FILES_OPEN=$(ulimit -n)
else
- $SSH $HOST "bash -c 'exec nohup ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
+ $SSH $HOST "bash -c 'exec nohup ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
MAX_FILES_OPEN=$($SSH $HOST "/usr/bin/env bash -c 'ulimit -n'")
fi
[5/7] accumulo git commit: Merge branch '1.7' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.7
Posted by ec...@apache.org.
Merge branch '1.7' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f122a4ac
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f122a4ac
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f122a4ac
Branch: refs/heads/master
Commit: f122a4ac5d88879ec95c319c55bd81b46dbf22d1
Parents: 9aee248 98f42c7
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:15:15 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:15:15 2015 -0400
----------------------------------------------------------------------
CHANGES | 703 +++++++++++++++++++
assemble/bin/build_native_library.sh | 6 +-
.../apache/accumulo/test/randomwalk/Node.java | 2 +-
3 files changed, 707 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[2/7] accumulo git commit: ACCUMULO-3793 use numctl to turn on memory
interleaving
Posted by ec...@apache.org.
ACCUMULO-3793 use numctl to turn on memory interleaving
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aee2489
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aee2489
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aee2489
Branch: refs/heads/master
Commit: 9aee2489b8b824444639223f4d2ac144501d4612
Parents: 1c6e4b6 b821487
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:14:17 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:14:17 2015 -0400
----------------------------------------------------------------------
assemble/bin/config.sh | 9 +++++++++
assemble/bin/start-server.sh | 4 ++--
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aee2489/assemble/bin/config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/config.sh
index d9bf8d4,9bee91b..ed393bb
--- a/assemble/bin/config.sh
+++ b/assemble/bin/config.sh
@@@ -104,6 -115,51 +104,15 @@@ the
fi
export HADOOP_PREFIX
-unset MASTER1
-if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then
- MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1)
-fi
-
-if [ -z "${MONITOR}" ] ; then
- MONITOR=$MASTER1
- if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then
- MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1)
- fi
- if [ -z "${MONITOR}" ] ; then
- echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty."
- exit 1
- fi
-fi
-if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then
- if [ -z "${MASTER1}" ] ; then
- echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty."
- exit 1
- else
- echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers"
- fi
-
-fi
-
-if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then
- if [ -z "${MASTER1}" ] ; then
- echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty."
- exit 1
- else
- echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc"
- fi
-fi
-
+ NUMA=`which numactl 2>/dev/null`
+ NUMACTL_EXISTS="$?"
+ NUMACTL_ARGS="--interleave=all"
+ if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then
+ export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}"
+ else
+ export NUMA_CMD=""
+ fi
+
-SSH='ssh -qnf -o ConnectTimeout=2'
-
export HADOOP_HOME=$HADOOP_PREFIX
export HADOOP_HOME_WARN_SUPPRESS=true
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aee2489/assemble/bin/start-server.sh
----------------------------------------------------------------------
diff --cc assemble/bin/start-server.sh
index c52bd64,daa8495..3587ec9
--- a/assemble/bin/start-server.sh
+++ b/assemble/bin/start-server.sh
@@@ -64,28 -66,23 +64,28 @@@ if [[ ${SERVICE} == "monitor" && ${ACCU
ADDRESS="0.0.0.0"
fi
-if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then
+if [[ $HOST == localhost || $HOST == "$(hostname -f)" || $HOST = "$IP" ]]; then
PID=$(ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk {'print $2'} | head -1)
else
- PID=$($SSH $HOST ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk {'print $2'} | head -1)
+ PID=$($SSH "$HOST" ps -ef | egrep "${ACCUMULO_HOME}/.*/accumulo.*.jar" | grep "Main $SERVICE" | grep -v grep | awk {'print $2'} | head -1)
fi
-if [ -z "$PID" ]; then
+if [[ -z "$PID" ]]; then
echo "Starting $LONGNAME on $HOST"
- if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then
- ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err &
+ COMMAND="${bin}/accumulo"
+ if [ "${ACCUMULO_WATCHER}" = "true" ]; then
+ COMMAND="${bin}/accumulo_watcher.sh ${LOGHOST}"
+ fi
+
+ if [[ $HOST == localhost || $HOST == "$(hostname -f)" || $HOST = "$IP" ]]; then
- "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out" 2>"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" &
++ ${NUMA_CMD} "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out" 2>"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" &
MAX_FILES_OPEN=$(ulimit -n)
else
- $SSH "$HOST" "bash -c 'exec nohup $COMMAND ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
- $SSH $HOST "bash -c 'exec nohup ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
- MAX_FILES_OPEN=$($SSH $HOST "/usr/bin/env bash -c 'ulimit -n'")
++ $SSH "$HOST" "bash -c 'exec nohup ${NUMA_CMD} $COMMAND ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &"
+ MAX_FILES_OPEN=$($SSH "$HOST" "/usr/bin/env bash -c 'ulimit -n'")
fi
- if [ -n "$MAX_FILES_OPEN" ] && [ -n "$SLAVES" ] ; then
+ if [[ -n $MAX_FILES_OPEN && -n $SLAVES ]] ; then
MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768}
if (( SLAVES > 10 )) && (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED ))
then
[4/7] accumulo git commit: Merge branch '1.6' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.6
Posted by ec...@apache.org.
Merge branch '1.6' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/83816af4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/83816af4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/83816af4
Branch: refs/heads/master
Commit: 83816af42fc0322240d5ddb5616a88a5fd9aeb2f
Parents: b821487 7813f50
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:15:02 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:15:02 2015 -0400
----------------------------------------------------------------------
assemble/bin/build_native_library.sh | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[7/7] accumulo git commit: Merge branch '1.7'
Posted by ec...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ce7943d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ce7943d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ce7943d4
Branch: refs/heads/master
Commit: ce7943d4abdb079da49f8b4b9c9c1b71b183356f
Parents: 65b384a 9bac8dd
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:15:34 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:15:34 2015 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[6/7] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ec...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9bac8dd7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9bac8dd7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9bac8dd7
Branch: refs/heads/master
Commit: 9bac8dd7d9159f239297862b05ee8b239822c2c5
Parents: f122a4a 83816af
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:15:20 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:15:20 2015 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[3/7] accumulo git commit: Merge branch '1.7'
Posted by ec...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/65b384ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/65b384ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/65b384ad
Branch: refs/heads/master
Commit: 65b384ad603753f39f39d5f4bb940e9209b03651
Parents: 5e05527 9aee248
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon May 11 14:14:42 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon May 11 14:14:42 2015 -0400
----------------------------------------------------------------------
assemble/bin/config.sh | 9 +++++++++
assemble/bin/start-server.sh | 4 ++--
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------