You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/07/15 16:23:44 UTC
[hadoop] branch ozone-0.4.1 updated (eaec8e2 -> 3c41bc7)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from eaec8e2 Revert "HDDS-1735. Create separate unit and integration test executor dev-support script"
new 97897b6 HDDS-1735. Create separate unit and integration test executor dev-support script
new 3c41bc7 HDDS-1800. Result of author check is inverted
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hadoop-ozone/dev-support/checks/acceptance.sh | 6 +++++-
hadoop-ozone/dev-support/checks/author.sh | 15 ++++++++++-----
hadoop-ozone/dev-support/checks/build.sh | 5 ++++-
hadoop-ozone/dev-support/checks/checkstyle.sh | 10 ++++++++--
hadoop-ozone/dev-support/checks/findbugs.sh | 12 +++++++-----
.../dev-support/checks/{unit.sh => integration.sh} | 12 ++++++++----
hadoop-ozone/dev-support/checks/isolation.sh | 7 +++++--
hadoop-ozone/dev-support/checks/rat.sh | 17 +++++++++++++----
.../dev-support/checks/{acceptance.sh => shellcheck.sh} | 15 ++++++++++++---
hadoop-ozone/dev-support/checks/unit.sh | 8 ++++----
10 files changed, 76 insertions(+), 31 deletions(-)
copy hadoop-ozone/dev-support/checks/{unit.sh => integration.sh} (72%)
copy hadoop-ozone/dev-support/checks/{acceptance.sh => shellcheck.sh} (64%)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDDS-1800. Result of author check is inverted
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3c41bc7f918e8e850966ef65a68cb7caa3dc009b
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Mon Jul 15 18:00:10 2019 +0200
HDDS-1800. Result of author check is inverted
Closes #1092
---
hadoop-ozone/dev-support/checks/author.sh | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh
index d5a469c..f50a396 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -20,9 +20,8 @@ cd "$DIR/../../.." || exit 1
AUTHOR="uthor"
AUTHOR="@a${AUTHOR}"
-grep -r --include="*.java" "$AUTHOR" .
if grep -r --include="*.java" "$AUTHOR" .; then
- exit 0
-else
exit 1
+else
+ exit 0
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDDS-1735. Create separate unit and integration
test executor dev-support script
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 97897b6ab747e973fd05feac70e4c89bce3daa07
Author: Márton Elek <el...@apache.org>
AuthorDate: Sat Jun 29 01:59:44 2019 +0200
HDDS-1735. Create separate unit and integration test executor dev-support script
(cherry picked from commit 62a057b8d647730498ea9a04d57f18b4520d09cf)
---
hadoop-ozone/dev-support/checks/acceptance.sh | 6 +++++-
hadoop-ozone/dev-support/checks/author.sh | 14 ++++++++++----
hadoop-ozone/dev-support/checks/build.sh | 5 ++++-
hadoop-ozone/dev-support/checks/checkstyle.sh | 10 ++++++++--
hadoop-ozone/dev-support/checks/findbugs.sh | 12 +++++++-----
.../dev-support/checks/{unit.sh => integration.sh} | 12 ++++++++----
hadoop-ozone/dev-support/checks/isolation.sh | 7 +++++--
hadoop-ozone/dev-support/checks/rat.sh | 17 +++++++++++++----
.../dev-support/checks/{acceptance.sh => shellcheck.sh} | 15 ++++++++++++---
hadoop-ozone/dev-support/checks/unit.sh | 8 ++++----
10 files changed, 76 insertions(+), 30 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index 8de920f..4a50e08 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export HADOOP_VERSION=3
-"$DIR/../../../hadoop-ozone/dist/target/ozone-*-SNAPSHOT/compose/test-all.sh"
+OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
+cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
+./test-all.sh
exit $?
diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh
index 43caa70..d5a469c 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -13,10 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-mkdir -p ./target
-grep -r --include="*.java" "@author" .
-if [ $? -gt 0 ]; then
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+#hide this tring to not confuse yetus
+AUTHOR="uthor"
+AUTHOR="@a${AUTHOR}"
+
+grep -r --include="*.java" "$AUTHOR" .
+if grep -r --include="*.java" "$AUTHOR" .; then
exit 0
else
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/build.sh b/hadoop-ozone/dev-support/checks/build.sh
index 6a7811e..1197330 100755
--- a/hadoop-ozone/dev-support/checks/build.sh
+++ b/hadoop-ozone/dev-support/checks/build.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export MAVEN_OPTS="-Xmx4096m"
-mvn -am -pl :hadoop-ozone-dist -P hdds -Dmaven.javadoc.skip=true -DskipTests clean install
+mvn -B -f pom.ozone.xml -Dmaven.javadoc.skip=true -DskipTests clean install
exit $?
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh
index 0d80fbc..c4de528 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -13,11 +13,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-mvn -fn checkstyle:check -am -pl :hadoop-ozone-dist -Phdds
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+mvn -B -fn checkstyle:check -f pom.ozone.xml
+
+#Print out the exact violations with parsing XML results with sed
+find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
if [[ $violations -gt 0 ]]; then
echo "There are $violations checkstyle violations"
- exit -1
+ exit 1
fi
exit 0
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh
index 1328492..545ad9f 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -13,6 +13,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
FINDBUGS_ALL_FILE=./target/findbugs-all.txt
@@ -20,15 +22,15 @@ mkdir -p ./target
rm "$FINDBUGS_ALL_FILE" || true
touch "$FINDBUGS_ALL_FILE"
-mvn -fn findbugs:check -Dfindbugs.failOnError=false -am -pl :hadoop-ozone-dist -Phdds
+mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
-find hadoop-ozone -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-bugs=$(cat "$FINDBUGS_ALL_FILE" | wc -l)
+bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
if [[ ${bugs} -gt 0 ]]; then
- exit -1
+ exit 1
else
exit 0
fi
diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/integration.sh
similarity index 72%
copy from hadoop-ozone/dev-support/checks/unit.sh
copy to hadoop-ozone/dev-support/checks/integration.sh
index d839f22..1aea3d1 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/integration.sh
@@ -13,12 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export MAVEN_OPTS="-Xmx4096m"
-mvn -fn test -am -pl :hadoop-ozone-dist -P hdds
-module_failed_tests=$(find "." -name 'TEST*.xml'\
- | xargs "grep" -l -E "<failure|<error"\
+mvn -B install -f pom.ozone.xml -DskipTests
+mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem
+module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
+ | xargs -0 -n1 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
- exit -1
+ exit 1
fi
exit 0
diff --git a/hadoop-ozone/dev-support/checks/isolation.sh b/hadoop-ozone/dev-support/checks/isolation.sh
index c8f2c62..1280235 100755
--- a/hadoop-ozone/dev-support/checks/isolation.sh
+++ b/hadoop-ozone/dev-support/checks/isolation.sh
@@ -13,12 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -v hadoop-hdds | wc -l )
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -c -v hadoop-hdds )
if [ "$hadooplines" == "0" ]; then
echo "Only ozone/hdds subprojects are changed"
exit 0
else
echo "Main hadoop projects are changed in an ozone patch."
echo "Please do it in a HADOOP/HDFS patch and test it with hadoop precommit tests"
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/hadoop-ozone/dev-support/checks/rat.sh
index aadb251..68ca56e 100755
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/hadoop-ozone/dev-support/checks/rat.sh
@@ -13,12 +13,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
mkdir -p target
-rm target/rat-aggregated.txt
-mvn -fn org.apache.rat:apache-rat-plugin:0.13:check -am -pl :hadoop-ozone-dist -Phdds
-grep -r --include=rat.txt "!????" | tee ./target/rat-aggregated.txt
+REPORT_FILE="$DIR/../../../target/rat-aggretaged.txt"
+mkdir -p "$(dirname "$REPORT_FILE")"
+
+cd hadoop-hdds || exit 1
+mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
+cd ../hadoop-ozone || exit 1
+mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
+
+cd "$DIR/../../.." || exit 1
+grep -r --include=rat.txt "!????" | tee "$REPORT_FILE"
if [ "$(cat target/rat-aggregated.txt)" ]; then
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/shellcheck.sh
similarity index 64%
copy from hadoop-ozone/dev-support/checks/acceptance.sh
copy to hadoop-ozone/dev-support/checks/shellcheck.sh
index 8de920f..1284acd 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/shellcheck.sh
@@ -14,6 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export HADOOP_VERSION=3
-"$DIR/../../../hadoop-ozone/dist/target/ozone-*-SNAPSHOT/compose/test-all.sh"
-exit $?
+cd "$DIR/../../.." || exit 1
+
+OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
+mkdir -p "$(dirname "$OUTPUT_FILE")"
+echo "" > "$OUTPUT_FILE"
+find "./hadoop-hdds" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
+find "./hadoop-ozone" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
+
+
+if [ "$(cat "$OUTPUT_FILE")" ]; then
+ exit 1
+fi
diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh
index d839f22..1c6f2e9 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -14,11 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
export MAVEN_OPTS="-Xmx4096m"
-mvn -fn test -am -pl :hadoop-ozone-dist -P hdds
-module_failed_tests=$(find "." -name 'TEST*.xml'\
- | xargs "grep" -l -E "<failure|<error"\
+mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem
+module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
+ | xargs -n1 -0 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
- exit -1
+ exit 1
fi
exit 0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org