You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/11/11 22:19:31 UTC
[15/17] yetus git commit: YETUS-15. build environment
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/build.sh
----------------------------------------------------------------------
diff --git a/build.sh b/build.sh
deleted file mode 100755
index a6f2dc9..0000000
--- a/build.sh
+++ /dev/null
@@ -1,283 +0,0 @@
-#!/bin/bash -e
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Temporary script for building tarballs. See YETUS-125 to help
-# create a more sustainable build system.
-#
-# Pass --release to get release checks
-# Pass --deploy to deploy maven snapshot artifacts
-#
-# Presumes you have
-# * maven 3.2.0+
-# * jdk 1.7+ (1.7 in --release)
-# * ruby + gems needed to run middleman
-# * python + python-dateutil
-
-## @description Verify that all required dependencies exist
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param true iff this is a release build
-## @return 1 - Some dependencies are missing
-## @return 0 - All dependencies exist
-function detect_dependencies
-{
- declare is_release=$1
- local exit_code=0
- if ! [ -x "$(command -v java)" ]; then
- echo "Java not found! Must install JDK version >= 1.7" >&2
- exit_code=1
- fi
- if ! [ -x "$(command -v mvn)" ]; then
- echo "Apache Maven not found! Must install version >= 3.2.0" >&2
- echo "Download it at https://maven.apache.org/download.cgi" >&2
- exit_code=1
- fi
- if ! [ -x "$(command -v bundle)" ]; then
- echo "building docs requires a Ruby executable bundle." >&2
- echo "Install it by executing 'gem install bundler && bundle install'" >&2
- exit_code=1
- fi
-
- ! python -c 'import dateutil.parser' 2>/dev/null
- if [ "$?" -eq "0" ]; then
- echo "Building release docs requires the python-dateutil module" >&2
- echo "Install it by executing 'pip install python-dateutil'" >&2
- exit_code=1
- fi
-
- if ! [ -x "$(command -v tar)" ]; then
- echo "Building archives requires the 'tar' command." >&2
- exit_code=1
- fi
-
- if [ "${is_release}" = "true" ] && ! [ -x "$(command -v pax)" ]; then
- echo "building the release source archive requires the 'pax' command." >&2
- exit_code=1
- fi
-
- if [[ "${exit_code}" -ne "0" ]]; then
- echo "Some dependencies are missing. Exit now." >&2
- fi
- return ${exit_code}
-}
-
-YETUS_VERSION=$(cat VERSION)
-RAT_DOWNLOAD_URL=https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.11/apache-rat-0.11.jar
-
-release=false
-offline=false
-for arg in "$@"; do
- case ${arg} in
- --release)
- release=true
- ;;
- --offline)
- offline=true
- ;;
- --deploy)
- deploy=true
- ;;
- esac
-done
-
-echo "working on version '${YETUS_VERSION}'"
-
-detect_dependencies "${release}"
-mkdir -p target
-
-if [ "${offline}" != "true" ]; then
- JIRA_VERSION="${YETUS_VERSION%%-SNAPSHOT}"
- echo "generating release docs."
- # Note that we use the bare python here instead of the wrapper script, since we
- # create said script.
- release-doc-maker/releasedocmaker.py --lint=all --license --outputdir target \
- --project YETUS "--version=${JIRA_VERSION}" \
- --projecttitle="Apache Yetus" --usetoday
-else
- echo "in offline mode, skipping release notes."
-fi
-
-MAVEN_ARGS=()
-if [ "${offline}" = "true" ]; then
- MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline)
-fi
-
-if [ "${release}" = "true" ]; then
- MAVEN_ARGS=("${MAVEN_ARGS[@]}" -Papache-release)
- echo "hard reseting working directory."
- git reset --hard HEAD
-
- if [ ! -f target/rat.jar ]; then
- if [ "${offline}" != "true" ]; then
- echo "downloading rat jar file to '$(pwd)/target/'"
- curl -o target/rat.jar "${RAT_DOWNLOAD_URL}"
- else
- echo "in offline mode, can't retrieve rat jar. will skip license check."
- fi
- fi
- echo "creating source tarball at '$(pwd)/target/'"
- rm "target/yetus-${YETUS_VERSION}-src".tar* 2>/dev/null || true
- pax -w -f "target/yetus-${YETUS_VERSION}-src.tar" -s "/target/yetus-${YETUS_VERSION}/" target/RELEASENOTES.md target/CHANGELOG.md
- current=$(basename "$(pwd)")
- #shellcheck disable=SC2038
- (cd ..; find "${current}" \( -name target -o -name publish -o -name .git \) -prune -o ! -type d -print | xargs pax -w -a -f "${current}/target/yetus-${YETUS_VERSION}-src.tar" -s "/${current}/yetus-${YETUS_VERSION}/")
- gzip "target/yetus-${YETUS_VERSION}-src.tar"
-fi
-
-echo "running maven builds for java components"
-# build java components
-if [[ "${deploy}" = true ]]; then
- mvncmd=deploy
-else
- mvncmd=install
-fi
-mvn "${MAVEN_ARGS[@]}" "${mvncmd}" --file yetus-project/pom.xml
-mvn "${MAVEN_ARGS[@]}" -Pinclude-jdiff-module "${mvncmd}" javadoc:aggregate --file audience-annotations-component/pom.xml
-
-echo "building documentation"
-# build docs after javadocs
-docs_out=$(cd asf-site-src && bundle exec middleman build)
-echo "${docs_out}"
-
-bin_tarball="target/bin-dir/yetus-${YETUS_VERSION}"
-echo "creating staging area for convenience binary at '$(pwd)/${bin_tarball}'"
-rm -rf "${bin_tarball}" 2>/dev/null || true
-mkdir -p "${bin_tarball}"
-
-for i in LICENSE NOTICE; do
- lines=$(grep -n 'Apache Yetus Source' "${i}" | cut -f1 -d:)
- if [[ -z "${lines}" ]]; then
- cp -p "${i}" "${bin_tarball}"
- else
- ((lines=lines-2))
- head -n "${lines}" "${i}" > "${bin_tarball}/${i}"
- fi
-done
-
-cp target/RELEASENOTES.md target/CHANGELOG.md "${bin_tarball}"
-cp -r asf-site-src/publish/documentation/in-progress "${bin_tarball}/docs"
-
-mkdir -p "${bin_tarball}/lib"
-cp VERSION "${bin_tarball}/lib/"
-
-mkdir -p "${bin_tarball}/lib/yetus-project"
-cp yetus-project/pom.xml "${bin_tarball}/lib/yetus-project/yetus-project-${YETUS_VERSION}.pom"
-
-mkdir -p "${bin_tarball}/lib/audience-annotations"
-cp audience-annotations-component/audience-annotations/target/audience-annotations-*.jar \
- audience-annotations-component/audience-annotations-jdiff/target/audience-annotations-jdiff-*.jar \
- "${bin_tarball}/lib/audience-annotations/"
-
-cp -r shelldocs "${bin_tarball}/lib/"
-
-cp -r release-doc-maker "${bin_tarball}/lib/"
-
-cp -r precommit "${bin_tarball}/lib/"
-ln -s test-patch.sh "${bin_tarball}/lib/precommit/qbt.sh"
-
-mkdir -p "${bin_tarball}/bin"
-
-# Make a special version of the shell wrapper for releasedocmaker
-# that maintains the ability to have '--lint' mean '--lint=all'
-cat >"${bin_tarball}/bin/releasedocmaker" <<EOF
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-args=()
-for arg in "\${@}"; do
- if [ "\${arg}" = "-n" ] || [ "\${arg}" = "--lint" ]; then
- args=("\${args[@]}" "--lint=all")
- else
- args=("\${args[@]}" "\${arg}")
- fi
-done
-
-exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmaker.py" "\${args[@]}"
-EOF
-chmod +x "${bin_tarball}/bin/releasedocmaker"
-
-for utility in shelldocs/shelldocs.py \
- precommit/docker-cleanup.sh \
- precommit/qbt.sh \
- precommit/smart-apply-patch.sh \
- precommit/test-patch.sh \
- precommit/jenkins/jenkins-admin.py
-do
- wrapper=${utility##*/}
- wrapper=${wrapper%.*}
- cat >"${bin_tarball}/bin/${wrapper}" <<EOF
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/${utility}" "\${@}"
-EOF
- chmod +x "${bin_tarball}/bin/${wrapper}"
-done
-
-bin_file="target/yetus-${YETUS_VERSION}-bin.tar.gz"
-echo "creating convenience binary in '$(pwd)/target'"
-rm "${bin_file}" 2>/dev/null || true
-tar -C "$(dirname "${bin_tarball}")" -czf "${bin_file}" "$(basename "${bin_tarball}")"
-
-if [ "${release}" = "true" ] && [ -f target/rat.jar ]; then
- echo "checking asf licensing requirements for source tarball '$(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz'."
- rm -rf target/source-unpack 2>/dev/null || true
- mkdir target/source-unpack
- tar -C target/source-unpack -xzf "target/yetus-${YETUS_VERSION}-src.tar.gz"
- java -jar target/rat.jar -E .rat-excludes -d target/source-unpack
-
- echo "checking asf licensing requirements for convenience binary '$(pwd)/${bin_file}'."
- rm -rf target/bin-unpack 2>/dev/null || true
- mkdir target/bin-unpack
- tar -C target/bin-unpack -xzf "${bin_file}"
- java -jar target/rat.jar -E .rat-excludes -d target/bin-unpack
-fi
-echo "All Done!"
-echo "Find your output at:"
-if [ "${release}" = "true" ] && [ -f "target/yetus-${YETUS_VERSION}-src.tar.gz" ]; then
- echo " $(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz"
-fi
-if [ -f "${bin_file}" ]; then
- echo " $(pwd)/${bin_file}"
-fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/hooks/build
----------------------------------------------------------------------
diff --git a/hooks/build b/hooks/build
new file mode 100755
index 0000000..21876dc
--- /dev/null
+++ b/hooks/build
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -n "${DOCKERFILE_PATH}" ]]; then
+ docker build -t "${IMAGE_NAME}" \
+ --build-arg DOCKER_TAG="${DOCKER_TAG}" \
+ --build-arg DOCKER_REPO="${DOCKER_REPO}" \
+ -f "${DOCKERFILE_PATH}"
+else
+ docker build -t "${IMAGE_NAME}" \
+ --build-arg DOCKER_TAG="${DOCKER_TAG}" \
+ --build-arg DOCKER_REPO="${DOCKER_REPO}" \
+ .
+fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..334c807
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,336 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>21</version>
+ <relativePath/>
+ <!-- no parent resolution -->
+ </parent>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>yetus-project</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ <description>Project-wide definitions for Apache Yetus</description>
+ <name>Apache Yetus Project</name>
+ <packaging>pom</packaging>
+
+ <url>https://yetus.apache.org</url>
+ <inceptionYear>2015</inceptionYear>
+ <mailingLists>
+ <mailingList>
+ <name>Apache Yetus Dev List</name>
+ <subscribe>dev-subscribe@yetus.apache.org</subscribe>
+ <unsubscribe>dev-unsubscribe@yetus.apache.org</unsubscribe>
+ <post>dev@yetus.apache.org</post>
+ <archive>https://mail-archives.apache.org/mod_mbox/yetus-dev/</archive>
+ </mailingList>
+ </mailingLists>
+
+ <properties>
+ <maven.min.version>3.2.0</maven.min.version>
+ <maven.api.version>3.2</maven.api.version>
+ <maven.plugin.api.version>3.5.4</maven.plugin.api.version>
+ <java.min.version>1.8</java.min.version>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <extra.enforcer.version>1.0-beta-9</extra.enforcer.version>
+
+ <build-helper-maven-plugin.version>3.0.0</build-helper-maven-plugin.version>
+ <checkstyle.version>8.14</checkstyle.version>
+ <commons.io.version>2.6</commons.io.version>
+ <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
+ <jdiff.version>1.0.9</jdiff.version>
+ <jython-compile-maven-plugin.version>2.0</jython-compile-maven-plugin.version>
+ <jython-shaded.version>2.7.1</jython-shaded.version>
+ <maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
+ <maven-project-info-reports-plugin.version>3.0.0</maven-project-info-reports-plugin.version>
+ <spotbugs-maven-plugin.version>3.1.7</spotbugs-maven-plugin.version>
+
+ <sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor>
+
+ </properties>
+
+ <scm>
+ <connection>scm:git:git://git.apache.org/yetus.git</connection>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/yetus.git</developerConnection>
+ <url>https://git-wip-us.apache.org/repos/asf?p=yetus.git</url>
+ </scm>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/YETUS</url>
+ </issueManagement>
+
+ <build>
+
+ <pluginManagement>
+ <plugins>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${exec-maven-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <version>${spotbugs-maven-plugin.version}</version>
+ </plugin>
+
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>validate</id>
+ <phase>validate</phase>
+ <configuration>
+ <configLocation>checkstyle.xml</configLocation>
+ <encoding>UTF-8</encoding>
+ <consoleOutput>true</consoleOutput>
+ <failsOnError>true</failsOnError>
+ <linkXRef>false</linkXRef>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <failOnError>false</failOnError>
+ </configuration>
+ </plugin>
+
+ <!-- plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin -->
+
+
+ </plugins>
+ </pluginManagement>
+
+
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>extra-enforcer-rules</artifactId>
+ <version>${extra.enforcer.version}</version>
+ </dependency>
+ </dependencies>
+ <!-- version set by parent -->
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <!-- The earliest maven version we verify builds for via ASF Jenkins -->
+ <!-- Might be obviated by move to gradle -->
+ <requireMavenVersion>
+ <version>[${maven.min.version},)</version>
+ <message>Maven is out of date.
+ Yetus requires at least version ${maven.min.version} of Maven to properly build from source.
+ You appear to be using an older version. You can use either "mvn -version" or
+ "mvn enforcer:display-info" to verify what version is active.
+ See the contributor guide on building for more information: ${project.url}/contribute/
+ </message>
+ </requireMavenVersion>
+ <!-- The earliest JVM version we verify builds for via ASF Jenkins -->
+ <requireJavaVersion>
+ <version>[${java.min.version},)</version>
+ <message>Java is out of date.
+ Yetus requires at least version ${java.min.version} of the JDK to properly build from source.
+ You appear to be using an older version. You can use either "mvn -version" or
+ "mvn enforcer:display-info" to verify what version is active.
+ See the contributor guide on building for more information: ${project.url}/contribute/
+ </message>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludeSubprojects>false</excludeSubprojects>
+ <excludes>
+ <exclude>.pylintrc</exclude>
+ <exclude>.rubocop.yml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>gpg2</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <configuration>
+ <executable>gpg2</executable>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>jdk1.8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ <version>1.8</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+ </profiles>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <reportSets>
+ <reportSet>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <inherited>false</inherited>
+ <reportSets>
+ <reportSet>
+ <id>aggregate</id>
+ <configuration>
+ <maxmemory>1024m</maxmemory>
+ <quiet>true</quiet>
+ <verbose>false</verbose>
+ <source>${maven.compile.source}</source>
+ <charset>${maven.compile.encoding}</charset>
+ <reportOutputDirectory>${project.build.directory}/site</reportOutputDirectory>
+ <destDir>yetus-project/api</destDir>
+ <doclet>org.apache.yetus.audience.tools.IncludePublicAnnotationsStandardDoclet</doclet>
+ <docletArtifacts>
+ <docletArtifact>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>audience-annotations</artifactId>
+ <version>${project.version}</version>
+ </docletArtifact>
+ </docletArtifacts>
+ <useStandardDocletOptions>true</useStandardDocletOptions>
+
+ <!-- switch on dependency-driven aggregation -->
+ <includeDependencySources>false</includeDependencySources>
+
+ <dependencySourceIncludes>
+ <!-- include ONLY dependencies I control -->
+ <dependencySourceInclude>org.apache.hadoop:hadoop-annotations</dependencySourceInclude>
+ </dependencySourceIncludes>
+
+ </configuration>
+ <reports>
+ <report>aggregate</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+
+ <!-- plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>checkstyle</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin -->
+ </plugins>
+ </reporting>
+
+ <modules>
+ <module>yetus-minimaven-plugin</module>
+ <module>yetus-assemblies</module>
+ <module>audience-annotations-component</module>
+ <module>precommit</module>
+ <module>releasedocmaker</module>
+ <module>shelldocs</module>
+ <module>asf-site-src</module>
+ <module>yetus-dist</module>
+ <module>yetus-maven-plugin</module>
+ </modules>
+
+</project>
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/README.md
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/README.md b/precommit/coprocs.d/README.md
deleted file mode 100755
index e960cad..0000000
--- a/precommit/coprocs.d/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#############################
-##
-## bash v4+ coproc routines
-##
-#############################
-
-# bash v3 and lower will treat coproc commands as syntax errors
-# therefore, ALL functions which activate coprocs are located
-# here
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/e_a_r_helper.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/e_a_r_helper.sh b/precommit/coprocs.d/e_a_r_helper.sh
deleted file mode 100755
index 3df8293..0000000
--- a/precommit/coprocs.d/e_a_r_helper.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-## @description helper function for echo_and_redirect
-## @audience private
-## @stability evolving
-## @replaceable no
-function e_a_r_helper
-{
- declare logfile=$1
- shift
- declare params=("${@}")
-
- echo "Launching yrr_coproc" >> "${COPROC_LOGFILE}"
- # shellcheck disable=SC2034
- coproc yrr_coproc {
- ulimit -Su "${PROC_LIMIT}"
- yetus_run_and_redirect "${logfile}" "${params[@]}"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/process_counter.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/process_counter.sh b/precommit/coprocs.d/process_counter.sh
deleted file mode 100755
index 4f944bb..0000000
--- a/precommit/coprocs.d/process_counter.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-function process_counter_coproc_start
-{
- if [[ "${OSTYPE}" = Linux && "${DOCKERMODE}" = true ]]; then
- # this is really only even remotely close to
- # accurate under Docker, for the time being.
-
- echo "Launching process_counter_coproc" >> "${COPROC_LOGFILE}"
- # shellcheck disable=SC2034
- coproc process_counter_coproc {
- declare threadcount
- declare maxthreadcount
- declare cmd
-
- sleep 2
- while true; do
- threadcount=$(ps -L -u "${USER_ID}" -o lwp 2>/dev/null | wc -l)
- if [[ ${threadcount} -gt ${maxthreadcount} ]]; then
- maxthreadcount="${threadcount}"
- echo "${maxthreadcount}" > "${PATCH_DIR}/threadcounter.txt"
- fi
- read -r -t 2 cmd
- case "${cmd}" in
- exit)
- exit 0
- ;;
- esac
- done
- }
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/reaper.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/reaper.sh b/precommit/coprocs.d/reaper.sh
deleted file mode 100755
index 4674130..0000000
--- a/precommit/coprocs.d/reaper.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-function reaper_coproc_start
-{
- if [[ "${REAPER_MODE}" != "off" ]]; then
-
- echo "Launching reaper_coproc" >> "${COPROC_LOGFILE}"
-
- # shellcheck disable=SC2034
- coproc reaper_coproc {
- reaper_coproc_func
- }
- fi
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/00-yetuslib.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/00-yetuslib.sh b/precommit/core.d/00-yetuslib.sh
deleted file mode 100755
index 983dfe6..0000000
--- a/precommit/core.d/00-yetuslib.sh
+++ /dev/null
@@ -1,331 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# we need to declare this globally as an array, which can only
-# be done outside of a function
-declare -a YETUS_OPTION_USAGE
-
-## @description Print a message to stderr
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function yetus_error
-{
- echo "$*" 1>&2
-}
-
-## @description Print a message to stderr if --debug is turned on
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function yetus_debug
-{
- if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
- echo "[$(date) DEBUG]: $*" 1>&2
- fi
-}
-
-## @description Given variable $1 delete $2 from it
-## @audience public
-## @stability stable
-## @replaceable no
-function yetus_delete_entry
-{
- if [[ ${!1} =~ \ ${2}\ ]] ; then
- yetus_debug "Removing ${2} from ${1}"
- eval "${1}"=\""${!1// ${2} }"\"
- fi
-}
-
-## @description Given variable $1 add $2 to it
-## @audience public
-## @stability stable
-## @replaceable no
-function yetus_add_entry
-{
- if [[ ! ${!1} =~ \ ${2}\ ]] ; then
- yetus_debug "Adding ${2} to ${1}"
- #shellcheck disable=SC2140
- eval "${1}"=\""${!1} ${2} "\"
- fi
-}
-
-## @description Given variable $1 determine if $2 is in it
-## @audience public
-## @stability stable
-## @replaceable no
-## @return 0 = yes, 1 = no
-function yetus_verify_entry
-{
- [[ ${!1} =~ \ ${2}\ ]]
-}
-
-## @description run the command, sending stdout and stderr to the given filename
-## @audience public
-## @stability stable
-## @param filename
-## @param command
-## @param [..]
-## @replaceable no
-## @return $?
-function yetus_run_and_redirect
-{
- declare logfile=$1
- shift
-
- # to the log
- {
- date
- echo "cd $(pwd)"
- echo "${*}"
- } >> "${logfile}"
- # run the actual command
- "${@}" >> "${logfile}" 2>&1
-}
-
-## @description Given a filename or dir, return the absolute version of it
-## @audience public
-## @stability stable
-## @param fsobj
-## @replaceable no
-## @return 0 success
-## @return 1 failure
-## @return stdout abspath
-function yetus_abs
-{
- declare obj=$1
- declare dir
- declare fn
-
- if [[ ! -e ${obj} ]]; then
- return 1
- elif [[ -d ${obj} ]]; then
- dir=${obj}
- else
- dir=$(dirname -- "${obj}")
- fn=$(basename -- "${obj}")
- fn="/${fn}"
- fi
-
- dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
- if [[ $? = 0 ]]; then
- echo "${dir}${fn}"
- return 0
- fi
- return 1
-}
-
-## @description Add a header to the usage output
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param header
-function yetus_add_header
-{
- declare text=$1
-
- #shellcheck disable=SC2034
- YETUS_USAGE_HEADER="${text}"
-}
-
-## @description Add an option to the usage output
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param subcommand
-## @param subcommanddesc
-function yetus_add_option
-{
- declare option=$1
- declare text=$2
-
- YETUS_OPTION_USAGE[${YETUS_OPTION_USAGE_COUNTER}]="${option}@${text}"
- ((YETUS_OPTION_USAGE_COUNTER=YETUS_OPTION_USAGE_COUNTER+1))
-}
-
-## @description Reset the usage information to blank
-## @audience private
-## @stability evolving
-## @replaceable no
-function yetus_reset_usage
-{
- # shellcheck disable=SC2034
- YETUS_OPTION_USAGE=()
- YETUS_OPTION_USAGE_COUNTER=0
-}
-
-## @description Print a screen-size aware two-column output
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param array
-function yetus_generic_columnprinter
-{
- declare -a input=("$@")
- declare -i i=0
- declare -i counter=0
- declare line
- declare text
- declare option
- declare giventext
- declare -i maxoptsize
- declare -i foldsize
- declare -a tmpa
- declare numcols
-
- if [[ -n "${COLUMNS}" ]]; then
- numcols=${COLUMNS}
- else
- numcols=$(tput cols) 2>/dev/null
- fi
-
- if [[ -z "${numcols}"
- || ! "${numcols}" =~ ^[0-9]+$ ]]; then
- numcols=75
- else
- ((numcols=numcols-5))
- fi
-
- while read -r line; do
- tmpa[${counter}]=${line}
- ((counter=counter+1))
- option=$(echo "${line}" | cut -f1 -d'@')
- if [[ ${#option} -gt ${maxoptsize} ]]; then
- maxoptsize=${#option}
- fi
- done < <(for text in "${input[@]}"; do
- echo "${text}"
- done | sort)
-
- i=0
- ((foldsize=numcols-maxoptsize))
-
- until [[ $i -eq ${#tmpa[@]} ]]; do
- option=$(echo "${tmpa[$i]}" | cut -f1 -d'@')
- giventext=$(echo "${tmpa[$i]}" | cut -f2 -d'@')
-
- while read -r line; do
- printf "%-${maxoptsize}s %-s\n" "${option}" "${line}"
- option=" "
- done < <(echo "${giventext}"| fold -s -w ${foldsize})
- ((i=i+1))
- done
-}
-
-## @description Convert a comma-delimited string to an array
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param arrayname
-## @param string
-function yetus_comma_to_array
-{
- declare var=$1
- declare string=$2
-
- oldifs="${IFS}"
- IFS=',' read -r -a "${var}" <<< "${string}"
- IFS="${oldifs}"
-}
-
-## @description Check if an array has a given value
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param element
-## @param array
-## @returns 0 = yes
-## @returns 1 = no
-function yetus_array_contains
-{
- declare element=$1
- shift
- declare val
-
- if [[ "$#" -eq 0 ]]; then
- return 1
- fi
-
- for val in "${@}"; do
- if [[ "${val}" == "${element}" ]]; then
- return 0
- fi
- done
- return 1
-}
-
-## @description Add the element if it is not
-## @description present in the given array
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param arrayname
-## @param element
-function yetus_add_array_element
-{
- declare arrname=$1
- declare add=$2
-
- declare arrref="${arrname}[@]"
- declare array=("${!arrref}")
-
- if ! yetus_array_contains "${add}" "${array[@]}"; then
- # shellcheck disable=SC1083,SC2086
- eval "${arrname}"=\(\"\${array[@]}\" \"${add}\" \)
- yetus_debug "$1 accepted $2"
- else
- yetus_debug "$1 declined $2"
- fi
-}
-
-## @description Sort an array by its elements
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param arrayvar
-function yetus_sort_array
-{
- declare arrname=$1
- declare arrref="${arrname}[@]"
- declare array=("${!arrref}")
-
- declare globstatus
- declare oifs
- declare -a sa
-
- globstatus=$(set -o | grep noglob | awk '{print $NF}')
-
- if [[ -n ${IFS} ]]; then
- oifs=${IFS}
- fi
- set -f
- # shellcheck disable=SC2034
- IFS=$'\n' sa=($(sort <<<"${array[*]}"))
- # shellcheck disable=SC1083
- eval "${arrname}"=\(\"\${sa[@]}\"\)
-
- if [[ -n "${oifs}" ]]; then
- IFS=${oifs}
- else
- unset IFS
- fi
-
- if [[ "${globstatus}" = off ]]; then
- set +f
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh
deleted file mode 100755
index e8dee09..0000000
--- a/precommit/core.d/01-common.sh
+++ /dev/null
@@ -1,615 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description Setup the default global variables
-## @audience public
-## @stability stable
-## @replaceable no
-function common_defaults
-{
- #shellcheck disable=SC2034
- BASEDIR=$(pwd)
- BUGSYSTEMS=""
- BUILDTOOL="nobuild"
- BUILDTOOLS=""
- #shellcheck disable=SC2034
- EXEC_MODES=""
- #shellcheck disable=SC2034
- JENKINS=false
- LOAD_SYSTEM_PLUGINS=true
- #shellcheck disable=SC2034
- OFFLINE=false
- OSTYPE=$(uname -s)
- #shellcheck disable=SC2034
- PATCH_BRANCH=""
- PATCH_BRANCH_DEFAULT="master"
- #shellcheck disable=SC2034
- PATCH_DRYRUNMODE=false
- PATCH_DIR=/tmp
- while [[ -e ${PATCH_DIR} ]]; do
- PATCH_DIR=/tmp/yetus-${RANDOM}.${RANDOM}
- done
- #shellcheck disable=SC2034
- PATCH_METHOD=""
- #shellcheck disable=SC2034
- PATCH_METHODS=("gitapply" "patchcmd")
- #shellcheck disable=SC2034
- PATCH_LEVEL=0
- #shellcheck disable=SC2034
- PATCH_SYSTEM=""
- PROJECT_NAME=unknown
- RESULT=0
- #shellcheck disable=SC2034
- ROBOT=false
- #shellcheck disable=SC2034
- SENTINEL=false
- #shellcheck disable=SC2034
- TESTTYPES=""
- TESTFORMATS=""
- USER_PLUGIN_DIR=""
-
- #shellcheck disable=SC2034
- YETUS_SHELL_SCRIPT_DEBUG=false
-
- # Solaris needs POSIX and GNU, not SVID
- case ${OSTYPE} in
- SunOS)
- AWK=${AWK:-/usr/xpg4/bin/awk}
- CURL=${CURL:-curl}
- DIFF=${DIFF:-/usr/gnu/bin/diff}
- FILE=${FILE:-file}
- GIT=${GIT:-git}
- GREP=${GREP:-/usr/xpg4/bin/grep}
- PATCH=${PATCH:-/usr/gnu/bin/patch}
- SED=${SED:-/usr/xpg4/bin/sed}
- ;;
- *)
- AWK=${AWK:-awk}
- CURL=${CURL:-curl}
- DIFF=${DIFF:-diff}
- FILE=${FILE:-file}
- GIT=${GIT:-git}
- GREP=${GREP:-grep}
- PATCH=${PATCH:-patch}
- SED=${SED:-sed}
- ;;
- esac
-
- RSYNC=${RSYNC:-rsync}
-}
-
-## @description Interpret the common command line parameters used by test-patch,
-## @description smart-apply-patch, and the bug system plugins
-## @audience private
-## @stability stable
-## @replaceable no
-## @param $@
-## @return May exit on failure
-function common_args
-{
- declare i
- declare showhelp=false
- declare showversion=false
- declare version
-
- for i in "$@"; do
- case ${i} in
- --awk-cmd=*)
- AWK=${i#*=}
- ;;
- --basedir=*)
- #shellcheck disable=SC2034
- BASEDIR=${i#*=}
- ;;
- --branch=*)
- #shellcheck disable=SC2034
- PATCH_BRANCH=${i#*=}
- ;;
- --branch-default=*)
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=${i#*=}
- ;;
- --curl-cmd=*)
- CURL=${i#*=}
- ;;
- --debug)
- #shellcheck disable=SC2034
- YETUS_SHELL_SCRIPT_DEBUG=true
- ;;
- --diff-cmd=*)
- DIFF=${i#*=}
- ;;
- --file-cmd=*)
- FILE=${i#*=}
- ;;
- --git-cmd=*)
- GIT=${i#*=}
- ;;
- --grep-cmd=*)
- GREP=${i#*=}
- ;;
- --help|-help|-h|help|--h|--\?|-\?|\?)
- showhelp=true
- ;;
- --list-plugins)
- list_plugins
- exit 0
- ;;
- --offline)
- #shellcheck disable=SC2034
- OFFLINE=true
- ;;
- --patch-cmd=*)
- PATCH=${i#*=}
- ;;
- --patch-dir=*)
- PATCH_DIR=${i#*=}
- ;;
- --plugins=*)
- ENABLED_PLUGINS=${i#*=}
- ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ }
- ;;
- --project=*)
- PROJECT_NAME=${i#*=}
- ;;
- --rsync-cmd=*)
- RSYNC=${i#*=}
- ;;
- --skip-system-plugins)
- LOAD_SYSTEM_PLUGINS=false
- ;;
- --sed-cmd=*)
- SED=${i#*=}
- ;;
- --user-plugins=*)
- USER_PLUGIN_DIR=${i#*=}
- ;;
- --version)
- showversion=true
- ;;
- *)
- ;;
- esac
- done
- if [[ ${showhelp} == true ]]; then
- yetus_usage
- exit 0
- fi
- if [[ ${showversion} == true ]]; then
- cat "${BINDIR}/../VERSION"
- exit 0
- fi
-
- # Absolutely require v1.7.3 or higher
- # versions lower than this either have bugs with
- # git apply or don't support all the
- # expected options
- version=$(${GIT} --version)
-
- if [[ $? != 0 ]]; then
- yetus_error "ERROR: ${GIT} failed during version detection."
- exit 1
- fi
-
- # shellcheck disable=SC2016
- version=$(echo "${version}" | ${AWK} '{print $NF}')
- if [[ ${version} =~ ^0
- || ${version} =~ ^1.[0-6]
- || ${version} =~ ^1.7.[0-2]$
- ]]; then
- yetus_error "ERROR: ${GIT} v1.7.3 or higher is required (found ${version})."
- exit 1
- fi
-}
-
-## @description List all installed plug-ins, regardless of whether
-## @description they have been enabled
-## @audience public
-## @stability evolving
-## @replaceable no
-function list_plugins
-{
- declare plugintype
- declare name
-
- ENABLED_PLUGINS="all"
- importplugins
-
- printf "Reminder: every plug-in may be enabled via 'all'.\n\n"
- for plugintype in BUILDTOOLS TESTTYPES BUGSYSTEMS TESTFORMATS; do
- printf "%s:\n\t" ${plugintype}
- for name in ${!plugintype}; do
- printf "%s " ${name}
- done
- echo ""
- done
-}
-
-## @description Let plugins also get a copy of the arguments
-## @audience private
-## @stability evolving
-## @replaceable no
-function parse_args_plugins
-{
- declare plugin
-
- for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do
- if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_parse_args"
- #shellcheck disable=SC2086
- ${plugin}_parse_args "$@"
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Initialize all enabled plugins
-## @audience private
-## @stability evolving
-## @replaceable no
-function plugins_initialize
-{
- declare plugin
-
- for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOL}; do
- if declare -f ${plugin}_initialize >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_initialize"
- #shellcheck disable=SC2086
- ${plugin}_initialize
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Determine if a plugin was enabled by the user
-## @description ENABLED_PLUGINS must be defined
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-function verify_plugin_enabled
-{
- declare toadd=$1
- declare bar
- declare idx
- declare strip
- declare stridx
-
- yetus_debug "Testing if $1 has been enabled by user"
-
- bar=""
- for idx in ${ENABLED_PLUGINS}; do
- stridx=${idx// }
- yetus_debug "verify_plugin_enabled: processing ${stridx}"
- case ${stridx} in
- all)
- bar=${toadd}
- ;;
- -*)
- strip=${stridx#-}
- if [[ ${strip} = "${toadd}" ]]; then
- bar=""
- fi
- ;;
- +*|*)
- strip=${stridx#+}
- if [[ ${strip} = "${toadd}" ]]; then
- bar=${toadd}
- fi
- ;;
- esac
- done
-
- if [[ -n ${bar} ]]; then
- yetus_debug "Post-parsing: checking ${bar} = ${toadd}"
- fi
- [[ ${bar} = "${toadd}" ]]
-}
-
-## @description Personality-defined plug-in list
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param plug-in list string
-function personality_plugins
-{
- if [[ -z "${ENABLED_PLUGINS}" ]]; then
- ENABLED_PLUGINS="$1"
- ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ }
- yetus_debug "Using personality plug-in list: ${ENABLED_PLUGINS}"
- fi
-}
-
-## @description Add the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-function add_test
-{
- if verify_plugin_enabled "${1}"; then
- yetus_add_entry NEEDED_TESTS "${1}"
- fi
-}
-
-## @description Remove the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-function delete_test
-{
- yetus_delete_entry NEEDED_TESTS "${1}"
-}
-
-## @description Verify if a given test was requested
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-## @return 0 = yes
-## @return 1 = no
-function verify_needed_test
-{
- yetus_verify_entry NEEDED_TESTS "${1}"
-}
-
-## @description Add the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param plugin
-function add_test_type
-{
- if verify_plugin_enabled "${1}"; then
- yetus_add_entry TESTTYPES "${1}"
- fi
-}
-
-## @description Remove the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param plugin
-function delete_test_type
-{
- yetus_delete_entry TESTTYPES "${1}"
-}
-
-## @description Add the given bugsystem type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param bugsystem
-function add_bugsystem
-{
- if verify_plugin_enabled "${1}"; then
- yetus_add_entry BUGSYSTEMS "${1}"
- fi
-}
-
-## @description Remove the given bugsystem type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param bugsystem
-function delete_bugsystem
-{
- yetus_delete_entry BUGSYSTEMS "${1}"
-}
-
-## @description Add the given test format type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test format
-function add_test_format
-{
- if verify_plugin_enabled "${1}"; then
- yetus_add_entry TESTFORMATS "${1}"
- fi
-}
-
-## @description Remove the given test format type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test format
-function delete_test_format
-{
- yetus_delete_entry TESTFORMATS "${1}"
-}
-
-## @description Add the given build tool type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param build tool
-function add_build_tool
-{
- if verify_plugin_enabled "${1}"; then
- yetus_add_entry BUILDTOOLS "${1}"
- fi
-}
-
-## @description Remove the given build tool type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param build tool
-function delete_build_tool
-{
- yetus_delete_entry BUILDTOOLS "${1}"
-}
-
-## @description Import content from test-patch.d and optionally
-## @description from user provided plugin directory
-## @audience private
-## @stability evolving
-## @replaceable no
-function importplugins
-{
- local i
- local plugin
- local files=()
-
- if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
- if [[ -d "${BINDIR}/test-patch.d" ]]; then
- files=(${BINDIR}/test-patch.d/*.sh)
- fi
- fi
-
- if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
- yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
- files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
- fi
-
- if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then
- yetus_error "ERROR: Can't find ${PERSONALITY} to import."
- unset PERSONALITY
- fi
-
- if [[ -z ${PERSONALITY}
- && -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
- && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
- yetus_debug "Using project personality."
- PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
- fi
-
- if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then
- yetus_debug "Importing ${PERSONALITY}"
- # shellcheck disable=SC1090
- . "${PERSONALITY}"
- fi
-
- for i in "${files[@]}"; do
- if [[ -f ${i} ]]; then
- yetus_debug "Importing ${i}"
- #shellcheck disable=SC1090
- . "${i}"
- fi
- done
-
- if declare -f personality_globals > /dev/null; then
- personality_globals
- fi
-}
-
-## @description Print the plugin's usage info
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param array
-function plugin_usage_output
-{
- echo ""
- echo "${YETUS_USAGE_HEADER}"
- echo ""
-}
-
-## @description Verifies the existence of a command
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param commandname
-## @param commandpath
-## @return 0 = ok
-## @return 1 = error
-function verify_command
-{
- local cmd_name="$1"
- local cmd_path="$2"
-
- if [[ -z ${cmd_path} ]]; then
- yetus_error "executable for '${cmd_name}' was not specified."
- return 1
- fi
- if [[ ! "${cmd_path}" =~ / ]]; then
- cmd_path=$(command -v "${cmd_path}")
- fi
- if [[ ! -f ${cmd_path} ]]; then
- yetus_error "executable '${cmd_path}' for '${cmd_name}' does not exist."
- return 1
- fi
- if [[ ! -x ${cmd_path} ]]; then
- yetus_error "executable '${cmd_path}' for '${cmd_name}' is not executable."
- return 1
- fi
- return 0
-}
-
-## @description Faster dirname, given the assumption that
-## @description dirs are always absolute (e.g., start with /)
-## @description DO NOT USE with relative paths or where
-## @description assumption may not be valid!
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param fileobj
-function faster_dirname
-{
- declare o=$1
-
- if [[ "${o}" =~ / ]]; then
- echo "${o%/*}"
- else
- echo .
- fi
-}
-
-## @description Set the USER_NAME, USER_ID, and GROUP_ID env vars
-## @audience private
-## @stability evolving
-## @replaceable no
-function determine_user
-{
- # On the Apache Jenkins hosts, $USER is pretty much untrustable because
- # something sets it to an account that doesn't actually exist.
- # Instead, we need to try and override it with something that's
- # probably close to reality.
- if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
- USER=$(id | cut -f2 -d\( | cut -f1 -d\))
- fi
-
- USER_NAME=${SUDO_USER:=$USER}
- # shellcheck disable=SC2034
- USER_ID=$(id -u "${USER_NAME}")
- # shellcheck disable=SC2034
- GROUP_ID=$(id -g "${USER_NAME}")
-}
-
-## @description Kill a process id
-## @audience private
-## @stability evolving
-## @replaceable yes
-## @param pid
-function pid_kill
-{
- declare pid=$1
- declare cmd
- declare kill_timeout=3
-
- kill "${pid}" >/dev/null 2>&1
- sleep "${kill_timeout}"
- if kill -0 "${pid}" > /dev/null 2>&1; then
- yetus_error "WARNING: ${pid} did not stop gracefully after ${kill_timeout} second(s): Trying to kill with kill -9"
- kill -9 "${pid}" >/dev/null 2>&1
- fi
- if ps -p "${pid}" > /dev/null 2>&1; then
- cmd=$(ps -o args "${pid}")
- yetus_error "ERROR: Unable to kill ${pid}: ${cmd}"
- fi
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh
deleted file mode 100755
index 0aeddf1..0000000
--- a/precommit/core.d/builtin-bugsystem.sh
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# This bug system handles the output on the screen.
-
-add_bugsystem console
-
-CONSOLE_USE_BUILD_URL=false
-
-## @description Print out the finished details on the console
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-## @return 0 on success
-## @return 1 on failure
-function console_finalreport
-{
- declare result=$1
- shift
- declare i=0
- declare ourstring
- declare vote
- declare subs
- declare ela
- declare comment
- declare commentfile1="${PATCH_DIR}/comment.1"
- declare commentfile2="${PATCH_DIR}/comment.2"
- declare normaltop
- declare line
- declare seccoladj=0
- declare spcfx=${PATCH_DIR}/spcl.txt
- declare calctime
-
- if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
- exec 6>&1
- exec >"${CONSOLE_REPORT_FILE}"
- fi
-
- if [[ ${result} == 0 ]]; then
- if [[ ${ROBOT} == false ]]; then
- if declare -f ${PROJECT_NAME}_console_success >/dev/null; then
- "${PROJECT_NAME}_console_success" > "${spcfx}"
- else
- {
- printf "IF9fX18gICAgICAgICAgICAgICAgICAgICAgICAgICAgICBfIAovIF9fX3wg";
- printf "XyAgIF8gIF9fXyBfX18gX19fICBfX18gX19ffCB8ClxfX18gXHwgfCB8IHwv";
- printf "IF9fLyBfXy8gXyBcLyBfXy8gX198IHwKIF9fXykgfCB8X3wgfCAoX3wgKF98";
- printf "ICBfXy9cX18gXF9fIFxffAp8X19fXy8gXF9fLF98XF9fX1xfX19cX19ffHxf";
- printf "X18vX19fKF8pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg";
- printf "ICAK";
- } > "${spcfx}"
- fi
- fi
- printf "\n\n+1 overall\n\n"
- else
- if [[ ${ROBOT} == false ]]; then
- if declare -f ${PROJECT_NAME}_console_failure >/dev/null; then
- "${PROJECT_NAME}_console_failure" > "${spcfx}"
- else
- {
- printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
- printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
- printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
- printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
- printf "ICAgICAgICAgICAgICAgICAK"
- } > "${spcfx}"
- fi
- fi
- printf "\n\n-1 overall\n\n"
- fi
-
- if [[ -f ${spcfx} ]]; then
- if which base64 >/dev/null 2>&1; then
- base64 --decode "${spcfx}" 2>/dev/null
- elif which openssl >/dev/null 2>&1; then
- openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
- fi
- echo
- echo
- rm "${spcfx}"
- fi
-
- seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}")
- if [[ ${seccoladj} -lt 10 ]]; then
- seccoladj=10
- fi
-
- seccoladj=$((seccoladj + 2 ))
- i=0
- until [[ $i -eq ${#TP_HEADER[@]} ]]; do
- printf "%s\n" "${TP_HEADER[${i}]}"
- ((i=i+1))
- done
-
- printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
- echo "============================================================================"
- i=0
- until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
- ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- ela=$(echo "${ourstring}" | cut -f4 -d\|)
- calctime=$(clock_display "${ela}")
- comment=$(echo "${ourstring}" | cut -f5 -d\|)
-
- echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
- normaltop=$(head -1 "${commentfile1}")
- ${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
-
- if [[ "${vote}" = "H" ]]; then
- printf "| | %*s | |%-s\n" ${seccoladj} " " "${normaltop}"
- else
- printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
- "${subs}" "${calctime}" "${normaltop}"
- fi
- while read -r line; do
- printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
- done < "${commentfile2}"
-
- ((i=i+1))
- rm "${commentfile2}" "${commentfile1}" 2>/dev/null
- done
-
- if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
- seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}")
- printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
- i=0
- until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
- ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
- ((i=i+1))
- done
- fi
-
- printf "\n\n|| Subsystem || Report/Notes ||\n"
- echo "============================================================================"
- i=0
-
- until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
- if [[ "${CONSOLE_USE_BUILD_URL}" = true &&
- -n "${BUILD_URL}" ]]; then
- comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
- else
- comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- fi
- printf "%s\n" "${comment}"
- ((i=i+1))
- done
-
- if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
- exec 1>&6 6>&-
- cat "${CONSOLE_REPORT_FILE}"
- fi
-}
-
-
-## @description Give access to the brief report file in docker mode
-## @audience private
-## @stability evolving
-## @replaceable no
-function console_docker_support
-{
- if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
- DOCKER_EXTRAARGS+=("-v" "${CONSOLE_REPORT_FILE}:/testptch/console.txt")
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-personality.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/builtin-personality.sh b/precommit/core.d/builtin-personality.sh
deleted file mode 100755
index c473d66..0000000
--- a/precommit/core.d/builtin-personality.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description Generate a list of all personality modules for a given
-## @description buildtool for the system to invoke
-## @audience private
-## @stability evolving
-## @replaceable no
-function personality_modules
-{
- if declare -f "${BUILDTOOL}_builtin_personality_modules" >/dev/null; then
- "${BUILDTOOL}_builtin_personality_modules" "$@"
- fi
-}
-
-## @description Generate a list of all personality file tests for a given
-## @description buildtool for the system to invoke
-## @audience private
-## @stability evolving
-## @replaceable no
-function personality_file_tests
-{
- if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
- "${BUILDTOOL}_builtin_personality_file_tests" "$@"
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh
deleted file mode 100755
index 83efc3b..0000000
--- a/precommit/core.d/docker.sh
+++ /dev/null
@@ -1,661 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DOCKERMODE=false
-DOCKERCMD=$(command -v docker)
-DOCKER_ID=${RANDOM}
-DOCKER_DESTRUCTIVE=true
-DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
-DOCKERFAIL="fallback,continue,fail"
-DOCKERSUPPORT=false
-DOCKER_ENABLE_PRIVILEGED=true
-DOCKER_CLEANUP_CMD=false
-DOCKER_MEMORY="4g"
-
-declare -a DOCKER_EXTRAARGS
-
-####
-#### IMPORTANT
-####
-#### If these times are updated, the documentation needs to
-#### be changed too!
-
-# created, stopped, exited, running, for 24 hours
-DOCKER_CONTAINER_PURGE=("86400" "86400" "86400" "86400" )
-
-# keep images for 1 week
-DOCKER_IMAGE_PURGE=604800
-
-## @description Docker-specific usage
-## @stability stable
-## @audience private
-## @replaceable no
-function docker_usage
-{
- if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
- yetus_add_option "--docker" "Spawn a docker container"
- fi
- yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')"
- if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
- yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')"
- yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})"
- yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')"
- fi
- yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them"
- if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
- yetus_add_option "--dockermemlimit=<num>" "Limit a Docker container's memory usage (default: ${DOCKER_MEMORY})"
- fi
-}
-
-## @description Docker-specific argument parsing
-## @stability stable
-## @audience private
-## @replaceable no
-## @params arguments
-function docker_parse_args
-{
- declare i
-
- for i in "$@"; do
- case ${i} in
- --docker)
- DOCKERSUPPORT=true
- ;;
- --dockercmd=*)
- #shellcheck disable=SC2034
- DOCKERCMD=${i#*=}
- ;;
- --dockerdelrep)
- DOCKER_DESTRUCTIVE=false
- ;;
- --dockerfile=*)
- DOCKERFILE=${i#*=}
- ;;
- --dockermemlimit=*)
- DOCKER_MEMORY=${i#*=}
- ;;
- --dockermode)
- DOCKERMODE=true
- ;;
- --dockeronfail=*)
- DOCKERFAIL=${i#*=}
- ;;
- --dockerprivd=*)
- DOCKER_ENABLE_PRIVILEGED=${i#*=}
- ;;
- esac
- done
-}
-
-## @description Docker initialization pre- and post- re-exec
-## @stability stable
-## @audience private
-## @replaceable no
-function docker_initialize
-{
- declare dockvers
-
- # --docker and --dockermode are mutually
- # exclusive. --docker is used by the user to
- # re-exec test-patch in Docker mode.
- # --dockermode is used by launch-test-patch (which is
- # run as the Docker EXEC in the Dockerfile,
- # see elsewhere for more info) to tell test-patch that
- # it has been restarted already. launch-test-patch
- # also strips --docker from the command line so that we
- # don't end up in a loop if the docker image
- # also has the docker command in it
-
- # we are already in docker mode
- if [[ "${DOCKERMODE}" == true ]]; then
- # DOCKER_VERSION is set by our creator.
- add_footer_table "Docker" "${DOCKER_VERSION}"
- return
- fi
-
- # docker mode hasn't been requested
- if [[ "${DOCKERSUPPORT}" != true ]]; then
- return
- fi
-
- # turn DOCKERFAIL into a string composed of numbers
- # to ease interpretation: 123, 213, 321, ... whatever
- # some of these combos are non-sensical but that's ok.
- # we'll treat non-sense as effectively errors.
- DOCKERFAIL=${DOCKERFAIL//,/ }
- DOCKERFAIL=${DOCKERFAIL//fallback/1}
- DOCKERFAIL=${DOCKERFAIL//continue/2}
- DOCKERFAIL=${DOCKERFAIL//fail/3}
- DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/}
-
- if ! docker_exeverify; then
- if [[ "${DOCKERFAIL}" =~ ^12
- || "${DOCKERFAIL}" =~ ^2 ]]; then
- add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker."
- DOCKERSUPPORT=false
- else
- add_vote_table -1 docker "Docker command '${DOCKERCMD}' not found/broken."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
- fi
-
- dockvers=$(docker_version Client)
- if [[ "${dockvers}" =~ ^0
- || "${dockvers}" =~ ^1\.[0-5]$ || "${dockvers}" =~ ^1\.[0-5]\. ]]; then
- if [[ "${DOCKERFAIL}" =~ ^12
- || "${DOCKERFAIL}" =~ ^2 ]]; then
- add_vote_table 0 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker."
- DOCKERSUPPORT=false
- else
- add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
- fi
-}
-
-## @description Verify dockerfile exists
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return exits on failure if configured
-function docker_fileverify
-{
- if [[ ${DOCKERMODE} = false &&
- ${DOCKERSUPPORT} = true ]]; then
- if [[ -n "${DOCKERFILE}" ]]; then
- pushd "${STARTINGDIR}" >/dev/null
- if [[ -f ${DOCKERFILE} ]]; then
- DOCKERFILE=$(yetus_abs "${DOCKERFILE}")
- else
- if [[ "${DOCKERFAIL}" =~ ^1 ]]; then
- yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, falling back to built-in."
- add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, falling back to built-in."
- DOCKERFILE=${DOCKERFILE_DEFAULT}
- elif [[ "${DOCKERFAIL}" =~ ^2 ]]; then
- yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, disabling docker."
- add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, disabling docker."
- DOCKERSUPPORT=false
- else
- yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found."
- add_vote_table -1 docker "Dockerfile '${DOCKERFILE}' not found."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
- fi
- popd >/dev/null
- else
- DOCKERFILE=${DOCKERFILE_DEFAULT}
- fi
- fi
-}
-
-## @description Verify docker exists
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 1 if docker is broken
-## @return 0 if docker is working
-function docker_exeverify
-{
- if ! verify_command "Docker" "${DOCKERCMD}"; then
- return 1
- fi
-
- if ! ${DOCKERCMD} info >/dev/null 2>&1; then
- yetus_error "Docker is not functioning properly. Daemon down/unreachable?"
- return 1
- fi
- return 0
-}
-
-## @description Run docker with some arguments, and
-## @description optionally send to debug.
-## @description some destructive commands require
-## @description DOCKER_DESTRUCTIVE to be set to true
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function dockercmd
-{
- declare subcmd=$1
- shift
-
- yetus_debug "dockercmd: ${DOCKERCMD} ${subcmd} $*"
- if [[ ${subcmd} == rm
- || ${subcmd} == rmi
- || ${subcmd} == stop
- || ${subcmd} == kill ]]; then
- if [[ "${DOCKER_DESTRUCTIVE}" == false ]]; then
- yetus_error "Safemode: not running ${DOCKERCMD} ${subcmd} $*"
- return
- fi
- fi
- "${DOCKERCMD}" "${subcmd}" "$@"
-}
-
-## @description Convet docker's time format to ctime
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param time
-function dockerdate_to_ctime
-{
- declare mytime=$1
-
- if [[ "${mytime}" = "0001-01-01T00:00:00Z" ]]; then
- mytime="1970-01-01T00:00:00"
- fi
-
- # believe it or not, date is not even close to standardized...
- if [[ $(uname -s) == Linux ]]; then
-
- # GNU date
- date -d "${mytime}" "+%s"
- else
-
- # BSD date; docker gives us two different format because fun
- if ! date -j -f "%FT%T%z" "${mytime}" "+%s" 2>/dev/null; then
- date -j -f "%FT%T" "${mytime}" "+%s"
- fi
- fi
-}
-
-## @description Stop and delete all defunct containers
-## @audience private
-## @stability evolving
-## @replaceable no
-function docker_container_maintenance
-{
- declare line
- declare id
- declare name
- declare status
- declare tmptime
- declare createtime
- declare starttime
- declare stoptime
- declare remove
- declare difftime
- declare data
-
- if [[ "${ROBOT}" = false ]]; then
- return
- fi
-
- big_console_header "Docker Container Maintenance"
-
- dockercmd ps -a
-
- data=$(dockercmd ps -qa)
-
- if [[ -z "${data}" ]]; then
- return
- fi
-
- while read -r line; do
- id=$(echo "${line}" | cut -f1 -d, )
- status=$(echo "${line}" | cut -f2 -d, )
- tmptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. )
- createtime=$(dockerdate_to_ctime "${tmptime}")
- tmptime=$(echo "${line}" | cut -f4 -d, | cut -f1 -d. )
- starttime=$(dockerdate_to_ctime "${tmptime}")
- tmptime=$(echo "${line}" | cut -f5 -d, | cut -f1 -d. )
- stoptime=$(dockerdate_to_ctime "${tmptime}")
- name=$(echo "${line}" | cut -f6 -d, )
- curtime=$("${AWK}" 'BEGIN {srand(); print srand()}')
- remove=false
-
- case ${status} in
- created)
- ((difftime = curtime - createtime))
- if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[0]} ]]; then
- remove=true
- fi
- ;;
- stopped)
- ((difftime = curtime - stoptime))
- if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[1]} ]]; then
- remove=true
- fi
- ;;
- exited | dead)
- ((difftime = curtime - stoptime))
- if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[2]} ]]; then
- remove=true
- fi
- ;;
- running)
- ((difftime = curtime - starttime))
- if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[3]}
- && "${SENTINEL}" = true ]]; then
- remove=true
- echo "Attempting to kill docker container ${name} [${id}]"
- dockercmd kill "${id}"
- fi
- ;;
- *)
- ;;
- esac
-
- if [[ "${remove}" == true ]]; then
- echo "Attempting to remove docker container ${name} [${id}]"
- dockercmd rm "${id}"
- fi
-
- done < <(
- # shellcheck disable=SC2086
- dockercmd inspect \
- --format '{{.Id}},{{.State.Status}},{{.Created}},{{.State.StartedAt}},{{.State.FinishedAt}},{{.Name}}' \
- ${data})
-}
-
-## @description Delete images after ${DOCKER_IMAGE_PURGE}
-## @audience private
-## @stability evolving
-## @replaceable no
-function docker_image_maintenance_helper
-{
- declare id
- declare tmptime
- declare createtime
- declare difftime
- declare name
-
- if [[ "${ROBOT}" = false ]]; then
- return
- fi
-
- if [[ -z "$*" ]]; then
- return
- fi
-
- for id in "$@"; do
- tmptime=$(dockercmd inspect --format '{{.Created}}' "${id}" | cut -f1 -d. )
- createtime=$(dockerdate_to_ctime "${tmptime}")
- curtime=$(date "+%s")
-
- ((difftime = curtime - createtime))
- if [[ ${difftime} -gt ${DOCKER_IMAGE_PURGE} ]]; then
- echo "Attempting to remove docker image ${id}"
- dockercmd rmi "${id}"
- fi
- done
-}
-
-
-## @description get sentinel-level docker images
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_get_sentinel_images
-{
- #shellcheck disable=SC2016
- dockercmd images \
- | tail -n +2 \
- | "${GREP}" -v hours \
- | "${AWK}" '{print $1":"$2}' \
- | "${GREP}" -v "<none>:<none>"
-}
-
-## @description Remove untagged/unused images
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_image_maintenance
-{
- declare id
-
- if [[ "${ROBOT}" = false ]]; then
- return
- fi
-
- big_console_header "Removing old images"
-
- dockercmd images
-
- echo "Untagged images:"
-
- #shellcheck disable=SC2046
- docker_image_maintenance_helper $(dockercmd images --filter "dangling=true" -q --no-trunc)
-
- echo "Apache Yetus images:"
-
- # removing this by image id doesn't always work without a force
- # in the situations that, for whatever reason, docker decided
- # to use the same image. this was a rare problem with older
- # releases of yetus. at some point, we should revisit this
- # in the mean time, we're going to reconstruct the
- # repostory:tag and send that to get removed.
-
- #shellcheck disable=SC2046,SC2016
- docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | grep tp- | ${AWK} '{print $1":"$2}')
- #shellcheck disable=SC2046,SC2016
- docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | ${GREP} -v hours | ${AWK} '{print $1":"$2}')
-
- if [[ "${SENTINEL}" = false ]]; then
- return
- fi
-
- echo "Other images:"
- #shellcheck disable=SC2046
- docker_image_maintenance_helper $(docker_get_sentinel_images)
-}
-
-
-## @description Perform pre-run maintenance to free up
-## @description resources. With --jenkins, it is a lot
-## @description more destructive.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_cleanup
-{
-
- docker_image_maintenance
-
- docker_container_maintenance
-}
-
-## @description Determine the revision of a dockerfile
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_getfilerev
-{
- ${GREP} 'YETUS_PRIVATE: gitrev=' \
- "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \
- | cut -f2 -d=
-}
-
-function docker_version
-{
- declare vertype=$1
- declare val
- declare ret
-
- # new version command
- val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null)
- ret=$?
-
- if [[ ${ret} != 0 ]];then
- # old version command
- val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ')
- fi
-
- echo "${val}"
-}
-
-## @description Start a test patch docker container
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_run_image
-{
- declare dockerfilerev
- declare baseimagename
- declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}"
- declare containername="yetus_tp-${DOCKER_ID}"
- declare client
- declare server
- declare retval
- declare elapsed
-
- dockerfilerev=$(docker_getfilerev)
-
- baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}"
-
- # make a base image, if it isn't available
- big_console_header "Building base image: ${baseimagename}"
- start_clock
- dockercmd build \
- -t "${baseimagename}" \
- "${PATCH_DIR}/precommit/test-patch-docker"
- retval=$?
-
- #shellcheck disable=SC2046
- elapsed=$(clock_display $(stop_clock))
-
- echo ""
- echo "Total Elapsed time: ${elapsed}"
- echo ""
-
- if [[ ${retval} != 0 ]]; then
- yetus_error "ERROR: Docker failed to build image."
- add_vote_table -1 docker "Docker failed to build ${baseimagename}."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
-
- big_console_header "Building ${BUILDMODE} image: ${patchimagename}"
- start_clock
- # using the base image, make one that is patch specific
- dockercmd build \
- -t "${patchimagename}" \
- - <<PatchSpecificDocker
-FROM ${baseimagename}
-LABEL org.apache.yetus=""
-LABEL org.apache.yetus.testpatch.patch="tp-${DOCKER_ID}"
-LABEL org.apache.yetus.testpatch.project=${PROJECT_NAME}
-RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} || true
-RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} || true
-RUN chown -R ${USER_NAME} /home/${USER_NAME} || true
-ENV HOME /home/${USER_NAME}
-USER ${USER_NAME}
-PatchSpecificDocker
-
- retval=$?
-
- #shellcheck disable=SC2046
- elapsed=$(clock_display $(stop_clock))
-
- echo ""
- echo "Total Elapsed time: ${elapsed}"
- echo ""
-
- if [[ ${retval} != 0 ]]; then
- yetus_error "ERROR: Docker failed to build image."
- add_vote_table -1 docker "Docker failed to build ${patchimagename}."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
-
- if [[ "${DOCKER_ENABLE_PRIVILEGED}" = true ]]; then
- DOCKER_EXTRAARGS+=("--privileged")
- fi
-
- if [[ -n "${DOCKER_MEMORY}" ]]; then
- DOCKER_EXTRAARGS+=("-m" "${DOCKER_MEMORY}")
- fi
-
- client=$(docker_version Client)
- server=$(docker_version Server)
-
- dockerversion="Client=${client} Server=${server}"
-
-
- # make the kernel prefer to kill us if we run out of RAM
- DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
-
- DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile")
- DOCKER_EXTRAARGS+=(-v "${PWD}:/testptch/${PROJECT_NAME}")
- DOCKER_EXTRAARGS+=(-u "${USER_NAME}")
- DOCKER_EXTRAARGS+=(-w "/testptch/${PROJECT_NAME}")
- DOCKER_EXTRAARGS+=("--env=BASEDIR=/testptch/${PROJECT_NAME}")
- DOCKER_EXTRAARGS+=("--env=DOCKER_VERSION=${dockerversion} Image:${baseimagename}")
- DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
- DOCKER_EXTRAARGS+=("--env=PATCH_SYSTEM=${PATCH_SYSTEM}")
- DOCKER_EXTRAARGS+=("--env=PROJECT_NAME=${PROJECT_NAME}")
- DOCKER_EXTRAARGS+=("--env=TESTPATCHMODE=${TESTPATCHMODE}")
- DOCKER_EXTRAARGS+=(--name "${containername}")
-
-
- trap 'docker_signal_handler' SIGTERM
- trap 'docker_signal_handler' SIGINT
-
- if [[ ${PATCH_DIR} =~ ^/ ]]; then
- dockercmd run --rm=true -i \
- "${DOCKER_EXTRAARGS[@]}" \
- -v "${PATCH_DIR}:/testptch/patchprocess" \
- --env=PATCH_DIR=/testptch/patchprocess \
- "${patchimagename}" &
- else
- dockercmd run --rm=true -i \
- "${DOCKER_EXTRAARGS[@]}" \
- --env=PATCH_DIR="${PATCH_DIR}" \
- "${patchimagename}" &
- fi
-
- wait ${!}
- cleanup_and_exit $?
-}
-
-## @description docker kill the container on SIGTERM
-## @audience private
-## @stability evolving
-## @replaceable no
-function docker_signal_handler
-{
- declare cid
-
- cid=$(cat "${PATCH_DIR}/cidfile")
-
- yetus_error "ERROR: Caught signal. Killing docker container:"
- dockercmd kill "${cid}"
- yetus_error "ERROR: Exiting."
- cleanup_and_exit 143 # 128 + 15 -- SIGTERM
-}
-
-## @description Switch over to a Docker container
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function docker_handler
-{
- PATCH_DIR=$(relative_dir "${PATCH_DIR}")
-
- docker_cleanup
- determine_user
- docker_run_image
-}