You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/07/03 18:11:35 UTC
[hadoop] branch checks2 created (now 06d125c)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch checks2
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
at 06d125c fix print0 consequences
This branch includes the following new commits:
new 215cbbf fixing ALL the checkstule issues under hadoop-ozone/dev-support/checks
new 06d125c fix print0 consequences
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: fix print0 consequences
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch checks2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 06d125c9abf5c0e1ba91347b500f5a9273b0112c
Author: Márton Elek <el...@apache.org>
AuthorDate: Wed Jul 3 20:10:55 2019 +0200
fix print0 consequences
---
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 4 ++--
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
.../checks/{integration.sh => shellcheck.sh} | 18 +++++++++---------
hadoop-ozone/dev-support/checks/unit.sh | 4 ++--
5 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh
index 19d6a89..c4de528 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -19,7 +19,7 @@ cd "$DIR/../../.." || exit 1
mvn -B -fn checkstyle:check -f pom.ozone.xml
#Print out the exact violations with parsing XML results with sed
-find "." -name checkstyle-errors.xml -print0 | xargs sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
+find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
if [[ $violations -gt 0 ]]; then
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh
index 5dcb650..545ad9f 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -24,8 +24,8 @@ touch "$FINDBUGS_ALL_FILE"
mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
-find hadoop-ozone -name findbugsXml.xml -print0 | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml -print0 | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh
index 9dac627..1aea3d1 100755
--- a/hadoop-ozone/dev-support/checks/integration.sh
+++ b/hadoop-ozone/dev-support/checks/integration.sh
@@ -20,7 +20,7 @@ export MAVEN_OPTS="-Xmx4096m"
mvn -B install -f pom.ozone.xml -DskipTests
mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs "grep" -l -E "<failure|<error"\
+ | xargs -0 -n1 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
exit 1
diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/shellcheck.sh
similarity index 67%
copy from hadoop-ozone/dev-support/checks/integration.sh
copy to hadoop-ozone/dev-support/checks/shellcheck.sh
index 9dac627..1284acd 100755
--- a/hadoop-ozone/dev-support/checks/integration.sh
+++ b/hadoop-ozone/dev-support/checks/shellcheck.sh
@@ -16,13 +16,13 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-export MAVEN_OPTS="-Xmx4096m"
-mvn -B install -f pom.ozone.xml -DskipTests
-mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs "grep" -l -E "<failure|<error"\
- | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
-if [[ -n "${module_failed_tests}" ]] ; then
- exit 1
+OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
+mkdir -p "$(dirname "$OUTPUT_FILE")"
+echo "" > "$OUTPUT_FILE"
+find "./hadoop-hdds" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
+find "./hadoop-ozone" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
+
+
+if [ "$(cat "$OUTPUT_FILE")" ]; then
+ exit 1
fi
-exit 0
diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh
index e9e09dd..35e5d9f 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
export MAVEN_OPTS="-Xmx4096m"
-mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem
+#mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs "grep" -l -E "<failure|<error"\
+ | xargs -n1 -0 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
exit 1
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: fixing ALL the checkstule issues under
hadoop-ozone/dev-support/checks
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch checks2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 215cbbf16913b648021504c9ad4f2a9ed3c285f6
Author: Márton Elek <el...@apache.org>
AuthorDate: Wed Jul 3 08:21:08 2019 +0200
fixing ALL the checkstule issues under hadoop-ozone/dev-support/checks
---
hadoop-ozone/dev-support/checks/acceptance.sh | 7 +++++--
hadoop-ozone/dev-support/checks/author.sh | 7 +++++--
hadoop-ozone/dev-support/checks/build.sh | 3 +++
hadoop-ozone/dev-support/checks/checkstyle.sh | 7 +++++--
hadoop-ozone/dev-support/checks/findbugs.sh | 10 ++++++----
hadoop-ozone/dev-support/checks/integration.sh | 7 +++++--
hadoop-ozone/dev-support/checks/isolation.sh | 7 +++++--
hadoop-ozone/dev-support/checks/rat.sh | 8 +++++---
hadoop-ozone/dev-support/checks/unit.sh | 4 ++--
9 files changed, 41 insertions(+), 19 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index 258c4e2..4a50e08 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -14,7 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export HADOOP_VERSION=3
-OZONE_VERSION=$(cat $DIR/../../pom.xml | grep "<ozone.version>" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
-"$DIR/../../dist/target/ozone-$OZONE_VERSION/compose/test-all.sh"
+OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
+cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
+./test-all.sh
exit $?
diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh
index 56d15a5..49a8fba 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -13,9 +13,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
grep -r --include="*.java" "@author" .
-if [ $? -gt 0 ]; then
+if grep -r --include="*.java" "@author" .; then
exit 0
else
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/build.sh b/hadoop-ozone/dev-support/checks/build.sh
index 71bf778..1197330 100755
--- a/hadoop-ozone/dev-support/checks/build.sh
+++ b/hadoop-ozone/dev-support/checks/build.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export MAVEN_OPTS="-Xmx4096m"
mvn -B -f pom.ozone.xml -Dmaven.javadoc.skip=true -DskipTests clean install
exit $?
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh
index 323cbc8..19d6a89 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -13,14 +13,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
mvn -B -fn checkstyle:check -f pom.ozone.xml
#Print out the exact violations with parsing XML results with sed
-find -name checkstyle-errors.xml | xargs sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
+find "." -name checkstyle-errors.xml -print0 | xargs sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
if [[ $violations -gt 0 ]]; then
echo "There are $violations checkstyle violations"
- exit -1
+ exit 1
fi
exit 0
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh
index c8bd40b..5dcb650 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -13,6 +13,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
FINDBUGS_ALL_FILE=./target/findbugs-all.txt
@@ -22,13 +24,13 @@ touch "$FINDBUGS_ALL_FILE"
mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
-find hadoop-ozone -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0 | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-bugs=$(cat "$FINDBUGS_ALL_FILE" | wc -l)
+bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
if [[ ${bugs} -gt 0 ]]; then
- exit -1
+ exit 1
else
exit 0
fi
diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh
index 8270d4f..9dac627 100755
--- a/hadoop-ozone/dev-support/checks/integration.sh
+++ b/hadoop-ozone/dev-support/checks/integration.sh
@@ -13,13 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export MAVEN_OPTS="-Xmx4096m"
mvn -B install -f pom.ozone.xml -DskipTests
mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem
-module_failed_tests=$(find "." -name 'TEST*.xml'\
+module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
| xargs "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
- exit -1
+ exit 1
fi
exit 0
diff --git a/hadoop-ozone/dev-support/checks/isolation.sh b/hadoop-ozone/dev-support/checks/isolation.sh
index c8f2c62..1280235 100755
--- a/hadoop-ozone/dev-support/checks/isolation.sh
+++ b/hadoop-ozone/dev-support/checks/isolation.sh
@@ -13,12 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -v hadoop-hdds | wc -l )
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -c -v hadoop-hdds )
if [ "$hadooplines" == "0" ]; then
echo "Only ozone/hdds subprojects are changed"
exit 0
else
echo "Main hadoop projects are changed in an ozone patch."
echo "Please do it in a HADOOP/HDFS patch and test it with hadoop precommit tests"
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/hadoop-ozone/dev-support/checks/rat.sh
index 9e16e12..43b4639 100755
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/hadoop-ozone/dev-support/checks/rat.sh
@@ -13,15 +13,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
mkdir -p target
rm target/rat-aggregated.txt
-cd hadoop-hdds
+cd hadoop-hdds || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
-cd ../hadoop-ozone
+cd ../hadoop-ozone || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
grep -r --include=rat.txt "!????" | tee ./target/rat-aggregated.txt
if [ "$(cat target/rat-aggregated.txt)" ]; then
- exit -1
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh
index fe12910..e9e09dd 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -15,10 +15,10 @@
# limitations under the License.
export MAVEN_OPTS="-Xmx4096m"
mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem
-module_failed_tests=$(find "." -name 'TEST*.xml'\
+module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
| xargs "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
- exit -1
+ exit 1
fi
exit 0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org