You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/09/12 10:25:00 UTC
[53/90] [abbrv] hadoop git commit: HDDS-222. Remove hdfs command line
from ozone distribution. Contributed by Elek, Marton.
HDDS-222. Remove hdfs command line from ozone distribution.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b5886bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b5886bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b5886bf
Branch: refs/heads/YARN-3409
Commit: 7b5886bf784579cc97656266901e6f934522b0e8
Parents: 7ce997a
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 11 10:19:50 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 11 10:19:50 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 6 +-
hadoop-hdds/client/pom.xml | 1 -
hadoop-hdds/container-service/pom.xml | 2 -
hadoop-hdds/framework/pom.xml | 1 -
hadoop-hdds/pom.xml | 3 -
hadoop-hdds/server-scm/pom.xml | 4 -
hadoop-ozone/common/src/main/bin/ozone | 8 +-
.../common/src/main/bin/ozone-config.sh | 51 +++++++++++
hadoop-ozone/common/src/main/bin/start-ozone.sh | 90 +-------------------
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 6 +-
hadoop-ozone/objectstore-service/pom.xml | 2 -
hadoop-ozone/ozone-manager/pom.xml | 2 -
hadoop-ozone/pom.xml | 9 --
13 files changed, 65 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index 1ba652c..d91c7ef 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -127,8 +127,6 @@ run cp -p "${ROOT}/README.txt" .
# Copy hadoop-common first so that it have always have all dependencies.
# Remaining projects will copy only libraries which are not present already in 'share' directory.
run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
-run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
-run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .
# HDDS
@@ -151,11 +149,15 @@ cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
+rm sbin/*all.sh
+rm sbin/*all.cmd
+
#Copy docker compose files
run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" .
mkdir -p ./share/hadoop/mapreduce
mkdir -p ./share/hadoop/yarn
+mkdir -p ./share/hadoop/hdfs
echo
echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-hdds/client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
index fb5320d..6a4cc9d 100644
--- a/hadoop-hdds/client/pom.xml
+++ b/hadoop-hdds/client/pom.xml
@@ -38,7 +38,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-hdds/container-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml
index c2b967c..3d4e581 100644
--- a/hadoop-hdds/container-service/pom.xml
+++ b/hadoop-hdds/container-service/pom.xml
@@ -37,12 +37,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-hdds/framework/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index 1d81b52..cb380e2 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -37,7 +37,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 0902c7d..563ce9e 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -44,17 +44,14 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-hdds/server-scm/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 11fcf88..52f42e1 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -37,25 +37,21 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-container-service</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 17b47a9..d14ccd9 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -144,11 +144,11 @@ fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
- # shellcheck source=./hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
- . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+if [[ -f "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" ]]; then
+ # shellcheck source=./hadoop-ozone/common/src/main/bin/ozone-config.sh
+ . "${HADOOP_LIBEXEC_DIR}/ozone-config.sh"
else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/ozone-config.sh." 2>&1
exit 1
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/common/src/main/bin/ozone-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone-config.sh b/hadoop-ozone/common/src/main/bin/ozone-config.sh
new file mode 100755
index 0000000..d179a33
--- /dev/null
+++ b/hadoop-ozone/common/src/main/bin/ozone-config.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# included in all the ozone scripts with source command
+# should not be executed directly
+
+function hadoop_subproject_init
+{
+ if [[ -z "${HADOOP_OZONE_ENV_PROCESSED}" ]]; then
+ if [[ -e "${HADOOP_CONF_DIR}/ozone-env.sh" ]]; then
+ . "${HADOOP_CONF_DIR}/ozone-env.sh"
+ export HADOOP_OZONE_ENV_PROCESSED=true
+ fi
+ fi
+ HADOOP_OZONE_HOME="${HADOOP_OZONE_HOME:-$HADOOP_HOME}"
+
+}
+
+if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
+ _hd_this="${BASH_SOURCE-$0}"
+ HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P)
+fi
+
+# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
+else
+ echo "ERROR: Hadoop common not found." 2>&1
+ exit 1
+fi
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index 29c3674..cfb54e0 100644
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -39,11 +39,11 @@ fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
+if [[ -f "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" ]]; then
# shellcheck disable=SC1090
- . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+ . "${HADOOP_LIBEXEC_DIR}/ozone-config.sh"
else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/ozone-config.sh." 2>&1
exit 1
fi
@@ -84,28 +84,6 @@ if [[ "${OZONE_ENABLED}" != "true" ]]; then
fi
#---------------------------------------------------------
-# Start hdfs before starting ozone daemons
-
-#---------------------------------------------------------
-# namenodes
-
-NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes 2>/dev/null)
-
-if [[ -z "${NAMENODES}" ]]; then
- NAMENODES=$(hostname)
-fi
-
-echo "Starting namenodes on [${NAMENODES}]"
-hadoop_uservar_su hdfs namenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${NAMENODES}" \
- --daemon start \
- namenode ${nameStartOpt}
-
-HADOOP_JUMBO_RETCOUNTER=$?
-
-#---------------------------------------------------------
# datanodes (using default workers file)
echo "Starting datanodes"
@@ -117,68 +95,6 @@ hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
#---------------------------------------------------------
-# secondary namenodes (if any)
-
-SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null)
-
-if [[ -n "${SECONDARY_NAMENODES}" ]]; then
-
- if [[ "${NAMENODES}" =~ , ]]; then
-
- hadoop_error "WARNING: Highly available NameNode is configured."
- hadoop_error "WARNING: Skipping SecondaryNameNode."
-
- else
-
- if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then
- SECONDARY_NAMENODES=$(hostname)
- fi
-
- echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
-
- hadoop_uservar_su hdfs secondarynamenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${SECONDARY_NAMENODES}" \
- --daemon start \
- secondarynamenode
- (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
- fi
-fi
-
-#---------------------------------------------------------
-# quorumjournal nodes (if any)
-
-JOURNAL_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -journalNodes 2>&-)
-
-if [[ "${#JOURNAL_NODES}" != 0 ]]; then
- echo "Starting journal nodes [${JOURNAL_NODES}]"
-
- hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${JOURNAL_NODES}" \
- --daemon start \
- journalnode
- (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
-fi
-
-#---------------------------------------------------------
-# ZK Failover controllers, if auto-HA is enabled
-AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]')
-if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
- echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]"
-
- hadoop_uservar_su hdfs zkfc "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${NAMENODES}" \
- --daemon start \
- zkfc
- (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
-fi
-
-#---------------------------------------------------------
# Ozone ozonemanager nodes
OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
echo "Starting Ozone Manager nodes [${OM_NODES}]"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index 5f5faf0..4321441 100644
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -39,11 +39,11 @@ fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
+if [[ -f "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" ]]; then
# shellcheck disable=SC1090
- . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+ . "${HADOOP_LIBEXEC_DIR}/ozone-config.sh"
else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/ozone-config.sh." 2>&1
exit 1
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/objectstore-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml
index 785f104..b79919f 100644
--- a/hadoop-ozone/objectstore-service/pom.xml
+++ b/hadoop-ozone/objectstore-service/pom.xml
@@ -38,13 +38,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/ozone-manager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index 5a312b2..3efb33b 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -38,13 +38,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5886bf/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index b61f041..09f3432 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -49,47 +49,38 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-scm</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-container-service</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-tools</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org