You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/06/06 11:00:22 UTC
[hadoop] branch trunk updated: HDDS-1458. Create a maven profile to
run fault injection tests. Contributed by Eric Yang.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f7c77b3 HDDS-1458. Create a maven profile to run fault injection tests. Contributed by Eric Yang.
f7c77b3 is described below
commit f7c77b395f6f01b4e687b9feb2e77b907d8e943f
Author: Márton Elek <el...@apache.org>
AuthorDate: Thu Jun 6 11:53:18 2019 +0200
HDDS-1458. Create a maven profile to run fault injection tests. Contributed by Eric Yang.
---
.../clusterUtils/__init__.py => .gitignore} | 8 +-
.../dist/dev-support/bin/dist-layout-stitching | 5 +-
.../fault-injection-test/network-tests/pom.xml | 103 +++++++++++++++++++++
.../network-tests/src/test}/blockade/README.md | 20 ++--
.../src/test/blockade/blockadeUtils}/__init__.py | 0
.../src/test}/blockade/blockadeUtils/blockade.py | 0
.../src/test}/blockade/clusterUtils/__init__.py | 0
.../test}/blockade/clusterUtils/cluster_utils.py | 2 +-
.../network-tests/src/test}/blockade/conftest.py | 0
.../src/test/blockade/ozone}/__init__.py | 0
.../src/test}/blockade/ozone/cluster.py | 20 ++--
.../test}/blockade/test_blockade_client_failure.py | 18 +++-
.../blockade/test_blockade_datanode_isolation.py | 3 +-
.../src/test}/blockade/test_blockade_flaky.py | 18 +++-
.../test}/blockade/test_blockade_mixed_failure.py | 18 +++-
...t_blockade_mixed_failure_three_nodes_isolate.py | 18 +++-
.../test_blockade_mixed_failure_two_nodes.py | 18 +++-
.../test}/blockade/test_blockade_scm_isolation.py | 18 +++-
.../network-tests/src/test}/blockade/util.py | 0
.../src/test/compose/docker-compose.yaml | 50 ++++++++++
.../network-tests/src/test/compose/docker-config | 77 +++++++++++++++
hadoop-ozone/fault-injection-test/pom.xml | 35 +++++++
hadoop-ozone/pom.xml | 2 +-
pom.ozone.xml | 2 +-
24 files changed, 391 insertions(+), 44 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py b/hadoop-ozone/.gitignore
similarity index 91%
copy from hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py
copy to hadoop-ozone/.gitignore
index 13878a1..93c6831 100644
--- a/hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py
+++ b/hadoop-ozone/.gitignore
@@ -11,4 +11,10 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
+
+*~
+*.pyc
+.blockade
+.cache
+__pycache__
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index 6178dfe..c9f81bd 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -83,6 +83,7 @@ run mkdir -p ./bin
run mkdir -p ./sbin
run mkdir -p ./etc
run mkdir -p ./libexec
+run mkdir -p ./tests
run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/om-audit-log4j2.properties" "etc/hadoop"
@@ -108,6 +109,9 @@ run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sb
run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/"
run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/"
+# fault injection tests
+run cp -r "${ROOT}/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade" tests
+
#shaded datanode service
run mkdir -p "./share/hadoop/ozoneplugin"
run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
@@ -119,7 +123,6 @@ cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./
#compose files are preprocessed: properties (eg. project.version) are replaced first by maven.
run cp -p -R "${ROOT}/hadoop-ozone/dist/target/compose" .
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
-run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" .
run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" .
diff --git a/hadoop-ozone/fault-injection-test/network-tests/pom.xml b/hadoop-ozone/fault-injection-test/network-tests/pom.xml
new file mode 100644
index 0000000..3b29480
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/network-tests/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-fault-injection-test</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-network-tests</artifactId>
+ <description>Apache Hadoop Ozone Network Tests</description>
+ <name>Apache Hadoop Ozone Network Tests</name>
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/test/compose</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>docker-compose.yaml</include>
+ <include>docker-config</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>it</id>
+ <properties>
+ <ozone.home>${basedir}../../dist/target/ozone-${project.version}</ozone.home>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>python</executable>
+ <arguments>
+ <argument>-m</argument>
+ <argument>pytest</argument>
+ <argument>-s</argument>
+ <argument>${basedir}/src/test/blockade/</argument>
+ </arguments>
+ <environmentVariables>
+ <OZONE_HOME>
+ ${ozone.home}
+ </OZONE_HOME>
+ <MAVEN_TEST>
+ ${project.build.directory}
+ </MAVEN_TEST>
+ </environmentVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
diff --git a/hadoop-ozone/dist/src/main/blockade/README.md b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md
similarity index 71%
rename from hadoop-ozone/dist/src/main/blockade/README.md
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md
index fb58205..b9f3c73 100644
--- a/hadoop-ozone/dist/src/main/blockade/README.md
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md
@@ -18,19 +18,25 @@ Following python packages need to be installed before running the tests :
1. blockade
2. pytest==2.8.7
+Running test as part of the maven build:
+
+mvn clean verify -Pit
+
+Running test as part of the released binary:
+
You can execute all blockade tests with following command-lines:
```
cd $DIRECTORY_OF_OZONE
-python -m pytest -s blockade/
+python -m pytest -s tests/blockade/
```
You can also execute fewer blockade tests with following command-lines:
```
cd $DIRECTORY_OF_OZONE
-python -m pytest -s blockade/<PATH_TO_PYTHON_FILE>
-e.g: python -m pytest -s blockade/test_blockade_datanode_isolation.py
+python -m pytest -s tests/blockade/<PATH_TO_PYTHON_FILE>
+e.g: python -m pytest -s tests/blockade/test_blockade_datanode_isolation.py
```
You can change the default 'sleep' interval in the tests with following
@@ -38,9 +44,9 @@ command-lines:
```
cd $DIRECTORY_OF_OZONE
-python -m pytest -s blockade/ --containerStatusSleep=<SECONDS>
+python -m pytest -s tests/blockade/ --containerStatusSleep=<SECONDS>
-e.g: python -m pytest -s blockade/ --containerStatusSleep=720
+e.g: python -m pytest -s tests/blockade/ --containerStatusSleep=720
```
By default, second phase of the tests will not be run.
@@ -49,6 +55,6 @@ command-lines:
```
cd $DIRECTORY_OF_OZONE
-python -m pytest -s blockade/ --runSecondPhase=true
+python -m pytest -s tests/blockade/ --runSecondPhase=true
-```
\ No newline at end of file
+```
diff --git a/hadoop-ozone/dist/src/main/blockade/ozone/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/__init__.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/ozone/__init__.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/__init__.py
diff --git a/hadoop-ozone/dist/src/main/blockade/blockadeUtils/blockade.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/blockade.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/blockadeUtils/blockade.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/blockade.py
diff --git a/hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/__init__.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/__init__.py
diff --git a/hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py
similarity index 99%
rename from hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py
index cf67380..53e3fa0 100644
--- a/hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py
@@ -332,4 +332,4 @@ class ClusterUtils(object):
datanodes = sorted(
list(filter(lambda x: 'datanode' in x, container_list)))
client = filter(lambda x: 'ozone_client' in x, container_list)
- return om, scm, client, datanodes
\ No newline at end of file
+ return om, scm, client, datanodes
diff --git a/hadoop-ozone/dist/src/main/blockade/conftest.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/conftest.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/conftest.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/conftest.py
diff --git a/hadoop-ozone/dist/src/main/blockade/blockadeUtils/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/__init__.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/blockadeUtils/__init__.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/__init__.py
diff --git a/hadoop-ozone/dist/src/main/blockade/ozone/cluster.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
similarity index 92%
rename from hadoop-ozone/dist/src/main/blockade/ozone/cluster.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
index 4347f86..f75b3d2 100644
--- a/hadoop-ozone/dist/src/main/blockade/ozone/cluster.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
@@ -21,6 +21,7 @@ import re
import subprocess
import yaml
import util
+from os import environ
from subprocess import call
from blockadeUtils.blockade import Blockade
@@ -44,11 +45,18 @@ class Configuration:
"""
def __init__(self):
- __parent_dir__ = os.path.dirname(os.path.dirname(
- os.path.dirname(os.path.realpath(__file__))))
- self.docker_compose_file = os.path.join(__parent_dir__,
- "compose", "ozoneblockade",
- "docker-compose.yaml")
+ if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ self.docker_compose_file = os.path.join(compose_dir, "docker-compose.yaml")
+ elif "OZONE_HOME" in os.environ:
+ compose_dir = os.path.join(environ.get("OZONE_HOME"), "compose", "ozoneblockade")
+ self.docker_compose_file = os.path.join(compose_dir, "docker-compose.yaml")
+ else:
+ __parent_dir__ = os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.dirname(os.path.realpath(__file__)))))
+ self.docker_compose_file = os.path.join(__parent_dir__,
+ "compose", "ozoneblockade",
+ "docker-compose.yaml")
self._datanode_count = 3
os.environ["DOCKER_COMPOSE_FILE"] = self.docker_compose_file
@@ -292,4 +300,4 @@ class Cluster(object):
container_states_dn = self.get_container_states(datanode)
if container_states_dn and container_states_dn.popitem()[1] == state:
return True
- return False
\ No newline at end of file
+ return False
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
similarity index 88%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
index 8c0b518..9e1b04f 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
@@ -19,14 +19,24 @@ import os
import re
import time
import logging
+from os import environ
from blockadeUtils.blockade import Blockade
from clusterUtils.cluster_utils import ClusterUtils
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 3
CONTAINER_LIST = []
@@ -118,4 +128,4 @@ def test_client_failure_isolate_one_datanode():
test_key_name, "/tmp/")
key_checksum = ClusterUtils.find_checksum(FILE, "/tmp/%s" % test_key_name)
- assert key_checksum == ORIG_CHECKSUM
\ No newline at end of file
+ assert key_checksum == ORIG_CHECKSUM
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
similarity index 99%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
index dfa1b70..85d99e2 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
@@ -22,7 +22,6 @@ from ozone.cluster import Cluster
logger = logging.getLogger(__name__)
-
def setup_function(function):
global cluster
cluster = Cluster.create()
@@ -135,4 +134,4 @@ def test_datanode_isolation_all():
util.wait_until(
lambda: cluster.container_state_predicate_all_closed(cluster.datanodes),
int(os.environ["CONTAINER_STATUS_SLEEP"]), 10)
- assert cluster.container_state_predicate_all_closed(cluster.datanodes)
\ No newline at end of file
+ assert cluster.container_state_predicate_all_closed(cluster.datanodes)
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py
similarity index 79%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py
index a79bd4f..6f1df18 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py
@@ -19,14 +19,24 @@ import os
import logging
import random
import pytest
+from os import environ
from blockadeUtils.blockade import Blockade
from ozone.cluster import Cluster
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 6
CONTAINER_LIST = []
@@ -64,4 +74,4 @@ def test_flaky(flaky_node):
Blockade.make_flaky(flaky_container_name)
Blockade.blockade_status()
exit_code, output = cluster.run_freon(1, 1, 1, 10240)
- assert exit_code == 0, "freon run failed with output=[%s]" % output
\ No newline at end of file
+ assert exit_code == 0, "freon run failed with output=[%s]" % output
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
similarity index 91%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
index 8493ce0..86d5311 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
@@ -19,13 +19,23 @@ import os
import time
import logging
import re
+from os import environ
from blockadeUtils.blockade import Blockade
from clusterUtils.cluster_utils import ClusterUtils
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 3
INCREASED_SCALE = 5
@@ -146,4 +156,4 @@ def test_one_dn_isolate_other_dn(run_second_phase):
"The container should have at least three closed replicas."
_, output = \
ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE")
- assert re.search("Status: Success", output) is not None
\ No newline at end of file
+ assert re.search("Status: Success", output) is not None
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
similarity index 94%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
index 0e50025..ab4c2d4 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
@@ -19,13 +19,23 @@ import os
import time
import logging
import re
+from os import environ
from blockadeUtils.blockade import Blockade
from clusterUtils.cluster_utils import ClusterUtils
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 3
INCREASED_SCALE = 5
@@ -222,4 +232,4 @@ def test_three_dns_isolate_threescmfailure(run_second_phase):
count_closed_container_datanodes = filter(
lambda x: x == 'CLOSED', all_datanodes_container_status)
assert len(count_closed_container_datanodes) == 3, \
- "The container should have three closed replicas."
\ No newline at end of file
+ "The container should have three closed replicas."
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
similarity index 92%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
index b8df2fa..03da7d0 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
@@ -19,13 +19,23 @@ import os
import time
import logging
import re
+from os import environ
from blockadeUtils.blockade import Blockade
from clusterUtils.cluster_utils import ClusterUtils
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 3
INCREASED_SCALE = 5
@@ -170,4 +180,4 @@ def test_two_dns_isolate_scm_different_partition(run_second_phase):
assert len(count_closed_container_datanodes) >= 3
_, output = \
ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE")
- assert re.search("Status: Success", output) is not None
\ No newline at end of file
+ assert re.search("Status: Success", output) is not None
diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
similarity index 91%
rename from hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
index 06f4263..47bbb76 100644
--- a/hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
@@ -19,13 +19,23 @@ import os
import time
import re
import logging
+from os import environ
from blockadeUtils.blockade import Blockade
from clusterUtils.cluster_utils import ClusterUtils
logger = logging.getLogger(__name__)
-parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-FILE = os.path.join(parent_dir, "compose", "ozoneblockade",
- "docker-compose.yaml")
+if "MAVEN_TEST" in os.environ:
+ compose_dir = environ.get("MAVEN_TEST")
+ FILE = os.path.join(compose_dir, "docker-compose.yaml")
+elif "OZONE_HOME" in os.environ:
+ compose_dir = environ.get("OZONE_HOME")
+ FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+else:
+ parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+ FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \
+ "docker-compose.yaml")
+
os.environ["DOCKER_COMPOSE_FILE"] = FILE
SCALE = 3
INCREASED_SCALE = 5
@@ -154,4 +164,4 @@ def test_scm_isolation_two_node(run_second_phase):
assert len(closed_container_datanodes) >= 3
_, output = \
ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE")
- assert re.search("Status: Success", output) is not None
\ No newline at end of file
+ assert re.search("Status: Success", output) is not None
diff --git a/hadoop-ozone/dist/src/main/blockade/util.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/util.py
similarity index 100%
rename from hadoop-ozone/dist/src/main/blockade/util.py
rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/util.py
diff --git a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml
new file mode 100644
index 0000000..6c8e0fb
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: ${user.name}/ozone:${project.version}
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ om:
+ image: ${user.name}/ozone:${project.version}
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: ${user.name}/ozone:${project.version}
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ ozone_client:
+ image: ${user.name}/ozone:${project.version}
+ ports:
+ - 9869
+ command: ["tail", "-f","/etc/passwd"]
+ env_file:
+ - ./docker-config
diff --git a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config
new file mode 100644
index 0000000..1db1a79
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.scm.dead.node.interval=5m
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
diff --git a/hadoop-ozone/fault-injection-test/pom.xml b/hadoop-ozone/fault-injection-test/pom.xml
new file mode 100644
index 0000000..395c534
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-fault-injection-test</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <description>Apache Hadoop Ozone Fault Injection Tests</description>
+ <name>Apache Hadoop Ozone Fault Injection Tests</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>network-tests</module>
+ </modules>
+
+</project>
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 2d80c3a..4a6df28 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -53,6 +53,7 @@
<module>ozone-recon-codegen</module>
<module>upgrade</module>
<module>csi</module>
+ <module>fault-injection-test</module>
</modules>
<repositories>
@@ -137,7 +138,6 @@
<version>${ozone.version}</version>
<type>test-jar</type>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
diff --git a/pom.ozone.xml b/pom.ozone.xml
index 413ca3b..ff841bd 100644
--- a/pom.ozone.xml
+++ b/pom.ozone.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main-ozone</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<description>Apache Hadoop Ozone Main</description>
<name>Apache Hadoop Ozone Main</name>
<packaging>pom</packaging>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org