You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/09/24 18:46:11 UTC
[47/50] [abbrv] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
HDDS-447. Separate ozone-dist and hadoop-dist projects with real classpath separation. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62f817d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62f817d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62f817d3
Branch: refs/heads/HDFS-12943
Commit: 62f817d32e6304b4cc1f9b909e7b815316b3a021
Parents: d060cbe
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Sep 24 10:10:11 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Sep 24 10:10:11 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 181 ------------
dev-support/bin/ozone-dist-tar-stitching | 43 ---
hadoop-dist/pom.xml | 118 --------
hadoop-dist/src/main/compose/README.md | 51 ----
hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 --
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 ----
.../src/main/compose/ozone-hdfs/docker-config | 76 -----
hadoop-dist/src/main/compose/ozone/.env | 17 --
.../src/main/compose/ozone/docker-compose.yaml | 49 ----
.../src/main/compose/ozone/docker-config | 75 -----
.../main/compose/ozonefs/docker-compose.yaml | 59 ----
.../src/main/compose/ozonefs/docker-config | 35 ---
hadoop-dist/src/main/compose/ozoneperf/.env | 17 --
.../src/main/compose/ozoneperf/README.md | 73 -----
.../src/main/compose/ozoneperf/compose-all.sh | 18 --
.../compose/ozoneperf/docker-compose-freon.yaml | 26 --
.../main/compose/ozoneperf/docker-compose.yaml | 64 -----
.../src/main/compose/ozoneperf/docker-config | 36 ---
hadoop-dist/src/main/compose/ozoneperf/init.sh | 21 --
.../src/main/compose/ozoneperf/prometheus.yml | 24 --
.../compose/ozonescripts/.ssh/authorized_keys | 16 --
.../src/main/compose/ozonescripts/.ssh/config | 18 --
.../main/compose/ozonescripts/.ssh/environment | 16 --
.../src/main/compose/ozonescripts/.ssh/id_rsa | 42 ---
.../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 --
.../src/main/compose/ozonescripts/Dockerfile | 33 ---
.../src/main/compose/ozonescripts/README.md | 38 ---
.../compose/ozonescripts/docker-compose.yaml | 42 ---
.../src/main/compose/ozonescripts/docker-config | 38 ---
hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 --
.../src/main/compose/ozonescripts/start.sh | 24 --
.../src/main/compose/ozonescripts/stop.sh | 17 --
hadoop-dist/src/main/ozone/README.txt | 51 ----
hadoop-dist/src/main/smoketest/README.md | 30 --
.../src/main/smoketest/basic/basic.robot | 47 ---
.../src/main/smoketest/basic/ozone-shell.robot | 82 ------
hadoop-dist/src/main/smoketest/commonlib.robot | 24 --
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 ---
hadoop-dist/src/main/smoketest/test.sh | 101 -------
hadoop-hdds/client/pom.xml | 5 -
hadoop-hdds/common/pom.xml | 2 -
hadoop-hdds/container-service/pom.xml | 5 -
hadoop-hdds/framework/pom.xml | 5 -
hadoop-hdds/pom.xml | 30 ++
hadoop-hdds/server-scm/pom.xml | 50 +---
.../src/main/resources/webapps/scm/index.html | 76 +++++
.../src/main/resources/webapps/scm/main.html | 20 ++
.../resources/webapps/scm/scm-overview.html | 60 ++++
.../src/main/resources/webapps/scm/scm.js | 54 ++++
.../server-scm/src/main/webapps/scm/index.html | 76 -----
.../server-scm/src/main/webapps/scm/main.html | 20 --
.../src/main/webapps/scm/scm-overview.html | 60 ----
.../server-scm/src/main/webapps/scm/scm.js | 54 ----
hadoop-hdds/tools/pom.xml | 8 -
hadoop-ozone/client/pom.xml | 6 -
hadoop-ozone/common/pom.xml | 5 -
hadoop-ozone/common/src/main/bin/ozone | 50 +++-
.../src/main/shellprofile.d/hadoop-ozone.sh | 24 --
hadoop-ozone/datanode/pom.xml | 49 ++++
.../dist/dev-support/bin/dist-layout-stitching | 113 ++++++++
.../dist/dev-support/bin/dist-tar-stitching | 45 +++
hadoop-ozone/dist/pom.xml | 184 ++++++++++++
hadoop-ozone/dist/src/main/compose/README.md | 51 ++++
.../dist/src/main/compose/ozone-hdfs/.env | 17 ++
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 ++++
.../src/main/compose/ozone-hdfs/docker-config | 76 +++++
hadoop-ozone/dist/src/main/compose/ozone/.env | 17 ++
.../src/main/compose/ozone/docker-compose.yaml | 49 ++++
.../dist/src/main/compose/ozone/docker-config | 75 +++++
.../main/compose/ozonefs/docker-compose.yaml | 59 ++++
.../dist/src/main/compose/ozonefs/docker-config | 35 +++
.../dist/src/main/compose/ozoneperf/.env | 17 ++
.../dist/src/main/compose/ozoneperf/README.md | 73 +++++
.../src/main/compose/ozoneperf/compose-all.sh | 18 ++
.../compose/ozoneperf/docker-compose-freon.yaml | 26 ++
.../main/compose/ozoneperf/docker-compose.yaml | 64 +++++
.../src/main/compose/ozoneperf/docker-config | 35 +++
.../dist/src/main/compose/ozoneperf/init.sh | 21 ++
.../src/main/compose/ozoneperf/prometheus.yml | 24 ++
.../compose/ozonescripts/.ssh/authorized_keys | 16 ++
.../src/main/compose/ozonescripts/.ssh/config | 18 ++
.../main/compose/ozonescripts/.ssh/environment | 16 ++
.../src/main/compose/ozonescripts/.ssh/id_rsa | 42 +++
.../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 ++
.../src/main/compose/ozonescripts/Dockerfile | 33 +++
.../src/main/compose/ozonescripts/README.md | 38 +++
.../compose/ozonescripts/docker-compose.yaml | 42 +++
.../src/main/compose/ozonescripts/docker-config | 38 +++
.../dist/src/main/compose/ozonescripts/ps.sh | 17 ++
.../dist/src/main/compose/ozonescripts/start.sh | 24 ++
.../dist/src/main/compose/ozonescripts/stop.sh | 17 ++
hadoop-ozone/dist/src/main/ozone/README.txt | 51 ++++
hadoop-ozone/dist/src/main/smoketest/README.md | 30 ++
.../dist/src/main/smoketest/basic/basic.robot | 47 +++
.../src/main/smoketest/basic/ozone-shell.robot | 82 ++++++
.../dist/src/main/smoketest/commonlib.robot | 24 ++
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 +++
hadoop-ozone/dist/src/main/smoketest/test.sh | 101 +++++++
hadoop-ozone/docs/pom.xml | 5 -
hadoop-ozone/integration-test/pom.xml | 14 -
.../hadoop/ozone/freon/TestDataValidate.java | 115 --------
.../ozone/freon/TestRandomKeyGenerator.java | 106 -------
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 -------------------
.../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ----------------
hadoop-ozone/objectstore-service/pom.xml | 5 -
hadoop-ozone/ozone-manager/pom.xml | 50 +---
.../resources/webapps/ozoneManager/index.html | 70 +++++
.../resources/webapps/ozoneManager/main.css | 23 ++
.../resources/webapps/ozoneManager/main.html | 18 ++
.../webapps/ozoneManager/om-metrics.html | 44 +++
.../webapps/ozoneManager/ozoneManager.js | 110 +++++++
.../src/main/webapps/ozoneManager/index.html | 70 -----
.../src/main/webapps/ozoneManager/main.css | 23 --
.../src/main/webapps/ozoneManager/main.html | 18 --
.../main/webapps/ozoneManager/om-metrics.html | 44 ---
.../main/webapps/ozoneManager/ozoneManager.js | 110 -------
hadoop-ozone/ozonefs/pom.xml | 2 -
hadoop-ozone/pom.xml | 25 +-
hadoop-ozone/tools/pom.xml | 32 ++-
.../hadoop/ozone/freon/TestDataValidate.java | 115 ++++++++
.../ozone/freon/TestRandomKeyGenerator.java | 106 +++++++
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 +++++++++++++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ++++++++++++++++
hadoop-project/pom.xml | 5 +
124 files changed, 3183 insertions(+), 3155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
deleted file mode 100755
index 00854b4..0000000
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# project.version
-VERSION=$1
-
-# project.build.directory
-BASEDIR=$2
-
-#hdds.version
-HDDS_VERSION=$3
-
-function run()
-{
- declare res
-
- echo "\$ ${*}"
- "${@}"
- res=$?
- if [[ ${res} != 0 ]]; then
- echo
- echo "Failed!"
- echo
- exit "${res}"
- fi
-}
-
-function findfileindir()
-{
- declare file="$1"
- declare dir="${2:-./share}"
- declare count
-
- count=$(find "${dir}" -iname "${file}" | wc -l)
-
- #shellcheck disable=SC2086
- echo ${count}
-}
-
-function copyifnotexists()
-{
- declare src="$1"
- declare dest="$2"
-
- declare srcname
- declare destdir
-
- declare child
- declare childpath
-
- if [[ -f "${src}" ]]; then
- srcname=${src##*/}
- if [[ "${srcname}" != *.jar ||
- $(findfileindir "${srcname}") -eq "0" ]]; then
- destdir=$(dirname "${dest}")
- mkdir -p "${destdir}"
- cp -p "${src}" "${dest}"
- fi
- else
- for childpath in "${src}"/*; do
- child="${childpath##*/}"
- if [[ "${child}" == "doc" ||
- "${child}" == "webapps" ]]; then
- mkdir -p "${dest}/${child}"
- cp -r "${src}/${child}"/* "${dest}/${child}"
- continue;
- fi
- copyifnotexists "${src}/${child}" "${dest}/${child}"
- done
- fi
-}
-
-#Copy all contents as is except the lib.
-#for libs check for existence in share directory, if not exist then only copy.
-function copy()
-{
- declare src="$1"
- declare dest="$2"
-
- declare child
- declare childpath
-
- if [[ -d "${src}" ]]; then
- for childpath in "${src}"/*; do
- child="${childpath##*/}"
-
- if [[ "${child}" == "share" ]]; then
- copyifnotexists "${src}/${child}" "${dest}/${child}"
- else
- if [[ -d "${src}/${child}" ]]; then
- mkdir -p "${dest}/${child}"
- cp -pr "${src}/${child}"/* "${dest}/${child}"
- else
- cp -pr "${src}/${child}" "${dest}/${child}"
- fi
- fi
- done
- fi
-}
-
-# shellcheck disable=SC2164
-ROOT=$(cd "${BASEDIR}"/../..;pwd)
-echo
-echo "Current directory $(pwd)"
-echo
-run rm -rf "ozone-${HDDS_VERSION}"
-run mkdir "ozone-${HDDS_VERSION}"
-run cd "ozone-${HDDS_VERSION}"
-run cp -p "${ROOT}/LICENSE.txt" .
-run cp -p "${ROOT}/NOTICE.txt" .
-
-# Copy hadoop-common first so that it have always have all dependencies.
-# Remaining projects will copy only libraries which are not present already in 'share' directory.
-run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
-
-
-# HDDS
-run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" .
-
-# Ozone
-run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
-
-#shaded ozonefs
-mkdir -p "./share/hadoop/ozonefs"
-cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
-
-#shaded datanode service
-mkdir -p "./share/hadoop/ozoneplugin"
-cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
-
-
-# Optional documentation, could be missing
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./
-
-
-rm sbin/*all.sh
-rm sbin/*all.cmd
-
-#remove test and java sources
-find . -name "*tests.jar" | xargs rm
-find . -name "*sources.jar" | xargs rm
-find . -name jdiff -type d | xargs rm -rf
-
-#add ozone specific readme
-
-run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt
-#Copy docker compose files and robot tests
-run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" .
-run cp -p -r "${ROOT}/hadoop-dist/src/main/smoketest" .
-
-mkdir -p ./share/hadoop/mapreduce
-mkdir -p ./share/hadoop/yarn
-mkdir -p ./share/hadoop/hdfs
-echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
-echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-tar-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-tar-stitching b/dev-support/bin/ozone-dist-tar-stitching
deleted file mode 100755
index 93d0525..0000000
--- a/dev-support/bin/ozone-dist-tar-stitching
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# project.version
-VERSION=$1
-
-# project.build.directory
-BASEDIR=$2
-
-function run()
-{
- declare res
-
- echo "\$ ${*}"
- "${@}"
- res=$?
- if [[ ${res} != 0 ]]; then
- echo
- echo "Failed!"
- echo
- exit "${res}"
- fi
-}
-
-run tar -c -f "ozone-${VERSION}.tar" "ozone-${VERSION}"
-run gzip -f "ozone-${VERSION}.tar"
-echo
-echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz"
-echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index da05015..5447b6a 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -176,7 +176,6 @@
</argument>
<argument>${project.version}</argument>
<argument>${project.build.directory}</argument>
- <argument>${hdds.version}</argument>
</arguments>
</configuration>
</execution>
@@ -224,123 +223,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-ozone-manager</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-scm</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-tools</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-container-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-objectstore-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-tools</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-docs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dockerfile</id>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <phase>prepare-package</phase>
- <configuration>
- <outputDirectory>${project.build.directory}</outputDirectory>
- <resources>
- <resource>
- <directory>src/main/docker</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>dist-ozone</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${shell-executable}</executable>
- <workingDirectory>${project.build.directory}
- </workingDirectory>
- <arguments>
- <argument>
- ${basedir}/../dev-support/bin/ozone-dist-layout-stitching
- </argument>
- <argument>${project.version}</argument>
- <argument>${project.build.directory}</argument>
- <argument>${hdds.version}</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>tar-ozone</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${shell-executable}</executable>
- <workingDirectory>${project.build.directory}
- </workingDirectory>
- <arguments>
- <argument>${basedir}/../dev-support/bin/ozone-dist-tar-stitching
- </argument>
- <argument>${hdds.version}</argument>
- <argument>${project.build.directory}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md
deleted file mode 100644
index 8189d2c..0000000
--- a/hadoop-dist/src/main/compose/README.md
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Docker cluster definitions
-
-This directory contains multiple docker cluster definitions to start local pseudo cluster with different configuration.
-
-It helps to start local (multi-node like) pseudo cluster with docker and docker-compose and obviously it's not for production.
-
-You may find more information in the specific subdirectories but in generic you can use the following commands:
-
-## Usage
-
-To start a cluster go to a subdirectory and start the cluster:
-
-```
-docker-compose up -d
-```
-
-You can check the logs of all the components with:
-
-```
-docker-compose logs
-```
-
-In case of a problem you can destroy the cluster an delete all the local state with:
-
-```
-docker-compose down
-```
-
-(Note: a simple docker-compose stop may not delete all the local data).
-
-You can scale up and down the components:
-
-```
-docker-compose scale datanode=5
-```
-
-Usually the key webui ports are published on the docker host.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env b/hadoop-dist/src/main/compose/ozone-hdfs/.env
deleted file mode 100644
index c437513..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HADOOP_VERSION=3.1.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
deleted file mode 100644
index b89052d..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- namenode:
- image: flokkr/hadoop:${HADOOP_VERSION}
- ports:
- - 9870:9870
- env_file:
- - ./docker-config
- environment:
- ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
- command: ["hdfs", "namenode"]
- datanode:
- image: flokkr/hadoop:${HADOOP_VERSION}
- ports:
- - 9864
- volumes:
- - ../..:/opt/ozone
- command: ["hdfs","datanode"]
- environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
deleted file mode 100644
index 3b2819f..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/.env b/hadoop-dist/src/main/compose/ozone/.env
deleted file mode 100644
index 67eed25..0000000
--- a/hadoop-dist/src/main/compose/ozone/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
deleted file mode 100644
index 0a6a9d8..0000000
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
deleted file mode 100644
index f2c8db1..0000000
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
deleted file mode 100644
index a1e8748..0000000
--- a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- hostname: ozoneManager
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
- hadooplast:
- image: flokkr/hadoop:3.1.0
- volumes:
- - ../..:/opt/ozone
- env_file:
- - ./docker-config
- environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
- command: ["watch","-n","100000","ls"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-config b/hadoop-dist/src/main/compose/ozonefs/docker-config
deleted file mode 100644
index 3171f08..0000000
--- a/hadoop-dist/src/main/compose/ozonefs/docker-config
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-dist/src/main/compose/ozoneperf/.env
deleted file mode 100644
index cac418a..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md
deleted file mode 100644
index 527ff41..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/README.md
+++ /dev/null
@@ -1,73 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Compose files for local performance tests
-
-This directory contains docker-compose definition for an ozone cluster where
-all the metrics are saved to a prometheus instance.
-
- Prometheus follows a pull based approach where the metrics are published
- on a HTTP endpoint.
-
- Our current approach:
-
- 1. A Java agent activates a prometheus metrics endpoint in every JVM instance
- (use `init.sh` to download the agent)
-
- 2. The Java agent publishes all the jmx parameters in prometheus format AND
- register the endpoint address to the consul.
-
- 3. Prometheus polls all the endpoints which are registered to consul.
-
-
-
-## How to use
-
-First of all download the required Java agent with running `./init.sh`
-
-After that you can start the cluster with docker-compose:
-
-```
-docker-compose up -d
-```
-
-After a while the cluster will be started. You can check the ozone web ui-s:
-
-https://localhost:9874
-https://localhost:9876
-
-You can also scale up the datanodes:
-
-```
-docker-compose scale datanode=3
-```
-
-Freon (Ozone test generator tool) is not part of docker-compose by default,
-you can activate it using `compose-all.sh` instead of `docker-compose`:
-
-```
-compose-all.sh up -d
-```
-
-Now Freon is running. Let's try to check the metrics from the local Prometheus:
-
-http://localhost:9090/graph
-
-Example queries:
-
-```
-Hadoop_OzoneManager_NumKeyCommits
-rate(Hadoop_OzoneManager_NumKeyCommits[10m])
-rate(Hadoop_Ozone_BYTES_WRITTEN[10m])
-```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
deleted file mode 100755
index 82ab8b3..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
deleted file mode 100644
index 60bdc4a..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- freon:
- image: apache/hadoop-runner
- volumes:
- - ../../ozone:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","freon"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
deleted file mode 100644
index 3b5cc7b..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
- consul:
- image: consul
- command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"]
- ports:
- - 8500:8500
- prometheus:
- image: prom/prometheus
- volumes:
- - "./prometheus.yml:/etc/prometheus.yml"
- command: ["--config.file","/etc/prometheus.yml"]
- ports:
- - 9090:9090
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config
deleted file mode 100644
index 454601e..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/init.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-dist/src/main/compose/ozoneperf/init.sh
deleted file mode 100755
index cf25398..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/init.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-EXPORTER_FILE="$DIR/jmxpromo.jar"
-if [ ! -f "$EXPORTER_FILE" ]; then
- wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE
-fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
deleted file mode 100644
index 80aa520..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-global:
- scrape_interval: 15s # By default, scrape targets every 15 seconds.
-
-scrape_configs:
- - job_name: jmxexporter
- consul_sd_configs:
- - server: consul:8500
- services:
- - jmxexporter
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
deleted file mode 100644
index ae39052..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
deleted file mode 100644
index 6506916..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Host *
- UserKnownHostsFile /dev/null
- StrictHostKeyChecking no
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
deleted file mode 100644
index 5685453..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
deleted file mode 100644
index 6632ce5..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
------BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEA4BJi6WJuAa1ratShvYYWVwmYBqxE57btHjU6NtVN1SnPZx/f
-6LezOpQGsLBXE/bl7uG+fD05Z378B/0wE5QhYwvJ9Ge0jsfhVOi90p/FEYfR2l+C
-9LRfLvO6AgA+HQa1BoYQd+norh/XQQVpcukSn32Cb642rW3OxsQTv3uTklDLapPw
-0lSpLJgYWcbrayorwjYw4rIfVYhiH/G+ckvzJRBpmlSgKYpNXyeYjpZaDPxoKQet
-GFf4W6nwnnG1oHojNYCvdoHTnAP1sdQu34/AUiOYg0hOnmXhfNIDN+NtZb8jw1sl
-YRWhm5GuK1wV/32/UEkQQPUU6oUSgBJgCH+PRwIDAQABAoIBAQDI1TH6ZNKchkck
-9XgSWsBjOqIcOQN5fCeDT8nho8WjLVpL3/Hcr+ngsxRcAXHK3xyvw33r9SQic1qJ
-/pC8u6RBFivo95qJ7vU0GXcp9TG4yLd6tui1U4WMm784U+dYNM7EDh1snSaECt3v
-1V3yNJ0QfnoOh2NShn0zAkOA+M4H8Nx2SudMCsjcbK9+fYxzW3hX+sJpMKdjG1HW
-DUz+I7cW7t0EGaVrgVSV+eR58LiXu+14YDNMrySiejB4nD2sKrx93XgiCBECCsBN
-GLQGJCztaXoAY+5Kf/aJ9EEf2wBF3GecRk+XIAd87PeDmeahLQAVkAJ/rD1vsKFs
-8kWi6CrhAoGBAP7leG/dMBhlfvROpBddIfWm2i8B+oZiAlSjdYGz+/ZhUaByXk18
-pupMGiiMQR1ziPnEg0gNgR2ZkH54qrXPn5WcQa4rlSEtUsZEp5v5WblhfX2QwKzY
-G/uhA+mB7wXpQkSmXo0LclfPF2teROQrG1OyfWkWbxFH4i3+em7sL95jAoGBAOEK
-v+wscqkMLW7Q8ONbWMCCBlmMHr6baB3VDCYZx25lr+GIF5zmJJFTmF2rq2VSAlts
-qx1AGmaUSo78kC5FuJvSNTL6a1Us5ucdthQZM3N8pAz+OAE+QEU+BsdA27yAh3tO
-yKDsMFNHKtXcgy5LeB5gzENLlNyw2jgkRv2Ef77NAoGAVH8DHqoHEH9Mx3XuRWR1
-JnaqKx0PzE5fEWmiQV3Fr5XxNivTgQJKXq7dmQVtbHLpPErdbhwz6fkHAjXD+UMb
-VsAWscL2y6m3n8wQd87/5EkiDWbXyDRXimGE53pQHviFJDa2bzEVNXCMBeaZFb4I
-cAViN1zdcrAOlUqfkXewIpsCgYB8wsXl/DpRB+RENGfn0+OfTjaQ/IKq72NIbq1+
-jfondQ6N/TICFQEe5HZrL9okoNOXteYjoD9CsWGoZdLVJGgVUvOVYImSvgMBDFK+
-T75bfzU/0sxfvBBLkviVDJsFpUf3D5VgybB86s6Po+HCD6r3RHjZshRESXOhflMx
-B3z+3QKBgE2Lwo0DuwUGkm6k8psyn3x8EiXNsiNw12cojicFTyKUYLHxMBeVbCLW
-3j3pxSggJgRuBLLzixUHbHp91r2ARTy28naK7R/la8yKVqK6ojcikN2mQsCHYtwB
-nuFwXr42ytn6G+9Wn4xT64tGjRCqyZn0/v0XsPjVCyrZ6G7EtNHP
------END RSA PRIVATE KEY-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
deleted file mode 100644
index ae39052..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
deleted file mode 100644
index 3bdcb0c..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM apache/hadoop-runner
-RUN sudo apt-get update && sudo apt-get install -y openssh-server
-
-RUN sudo mkdir -p /run/sshd
-RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
-RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config
-RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-#/opt/hadoop is mounted, we can't use it as a home
-RUN sudo usermod -d /opt hadoop
-ADD .ssh /opt/.ssh
-RUN sudo chown -R hadoop /opt/.ssh
-RUN sudo chown hadoop /opt
-RUN sudo chmod 600 /opt/.ssh/*
-RUN sudo chmod 700 /opt/.ssh
-
-RUN sudo sh -c 'echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/" >> /etc/profile'
-CMD ["sudo","/usr/sbin/sshd","-D"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/README.md b/hadoop-dist/src/main/compose/ozonescripts/README.md
deleted file mode 100644
index 2531fa4..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# start-ozone environment
-
-This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
-
-There are ssh connections between the containers and the start/stop scripts could handle the start/stop process
-similar to a real cluster.
-
-To use it, first start the cluster:
-
-```
-docker-copmose up -d
-```
-
-After a successfull startup (which starts only the ssh daemons) you can start ozone:
-
-```
-./start.sh
-```
-
-Check it the java processes are started:
-
-```
-./ps.sh
-```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
deleted file mode 100644
index 62f1163..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- env_file:
- - ./docker-config
- om:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- env_file:
- - ./docker-config
- scm:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-config b/hadoop-dist/src/main/compose/ozonescripts/docker-config
deleted file mode 100644
index 1afec73..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/docker-config
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
-OZONE-SITE.XML_ozone.ksm.address=ksm
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/ps.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/ps.sh b/hadoop-dist/src/main/compose/ozonescripts/ps.sh
deleted file mode 100755
index d5e2c38..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/ps.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/start.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/start.sh b/hadoop-dist/src/main/compose/ozonescripts/start.sh
deleted file mode 100755
index 3358b07..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/start.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1 docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
-docker-compose exec scm /opt/hadoop/bin/ozone scm -init
-docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
-#We need a running SCM for om objectstore creation
-#TODO create a utility to wait for the startup
-sleep 10
-docker-compose exec om /opt/hadoop/bin/ozone om -createObjectStore
-docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/stop.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/stop.sh b/hadoop-dist/src/main/compose/ozonescripts/stop.sh
deleted file mode 100755
index a3ce08a..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/stop.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose exec scm /opt/hadoop/sbin/stop-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/ozone/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/ozone/README.txt b/hadoop-dist/src/main/ozone/README.txt
deleted file mode 100644
index 6bbd83f..0000000
--- a/hadoop-dist/src/main/ozone/README.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-This is the distribution of Apache Hadoop Ozone.
-
-Ozone is a submodule of Hadoop with separated release cycle. For more information, check
-
- http://ozone.hadoop.apache.org
-
- and
-
- https://cwiki.apache.org/confluence/display/HADOOP/Ozone+Contributor+Guide
-
-For more information about Hadoop, check:
-
- http://hadoop.apache.org
-
-This distribution includes cryptographic software. The country in
-which you currently reside may have restrictions on the import,
-possession, use, and/or re-export to another country, of
-encryption software. BEFORE using any encryption software, please
-check your country's laws, regulations and policies concerning the
-import, possession, or use, and re-export of encryption software, to
-see if this is permitted. See <http://www.wassenaar.org/> for more
-information.
-
-The U.S. Government Department of Commerce, Bureau of Industry and
-Security (BIS), has classified this software as Export Commodity
-Control Number (ECCN) 5D002.C.1, which includes information security
-software using or performing cryptographic functions with asymmetric
-algorithms. The form and manner of this Apache Software Foundation
-distribution makes it eligible for export under the License Exception
-ENC Technology Software Unrestricted (TSU) exception (see the BIS
-Export Administration Regulations, Section 740.13) for both object
-code and source code.
-
-The following provides more details on the included cryptographic
-software:
- Hadoop Core uses the SSL libraries from the Jetty project written
-by mortbay.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/README.md b/hadoop-dist/src/main/smoketest/README.md
deleted file mode 100644
index c521a54..0000000
--- a/hadoop-dist/src/main/smoketest/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-## Ozone Acceptance Tests
-
-This directory contains a [robotframework](http://robotframework.org/) based test suite for Ozone to make it easier to check the current state of the package.
-
-You can run in in any environment after [installing](https://github.com/robotframework/robotframework/blob/master/INSTALL.rst)
-
-```
-cd $DIRECTORY_OF_OZONE
-robot smoketest/basic
-```
-
-The argument of the `robot` could be any robot file or directory.
-
-The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
-
-The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/basic/basic.robot
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/basic/basic.robot b/hadoop-dist/src/main/smoketest/basic/basic.robot
deleted file mode 100644
index a69450d..0000000
--- a/hadoop-dist/src/main/smoketest/basic/basic.robot
+++ /dev/null
@@ -1,47 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Smoketest ozone cluster startup
-Library OperatingSystem
-Resource ../commonlib.robot
-
-*** Variables ***
-${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${DATANODE_HOST} localhost
-
-
-*** Test Cases ***
-
-Test rest interface
- ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
- Should contain ${result} 201 Created
- ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
- Should contain ${result} 201 Created
- ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
- Should contain ${result} 200 OK
- ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
- Should contain ${result} 200 OK
-
-Check webui static resources
- ${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
- Should contain ${result} 200
- ${result} = Execute curl -s -I http://ozoneManager:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
- Should contain ${result} 200
-
-Start freon testing
- ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
- Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
- Should Not Contain ${result} ERROR
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org