You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by za...@apache.org on 2021/10/24 20:12:39 UTC
[hive] branch master updated: HIVE-25632: Remove unused code from
ptest/ptest2 framework (Stamatis Zampetakis, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fc349e7 HIVE-25632: Remove unused code from ptest/ptest2 framework (Stamatis Zampetakis, reviewed by Zoltan Haindrich)
fc349e7 is described below
commit fc349e719f840b5b863e5d4d4a1b2a31449709e9
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Thu Oct 21 13:03:19 2021 +0200
HIVE-25632: Remove unused code from ptest/ptest2 framework (Stamatis Zampetakis, reviewed by Zoltan Haindrich)
Ptest framework was deprecated when PTest2 was introduced, and the
latter is no longer used since it was superseded by HIVE-22942.
The code is more or less dead and keeping it in the repo leads to
maintenance overhead.
Drop also the following files since they are not used either and have
dependencies with ptests:
* jenkins-execute-build.sh
* jenkins-execute-hms-test.sh
* execute-test-on-lxc.sh
Closes #2736
---
dev-support/jenkins-execute-build.sh | 144 --
dev-support/jenkins-execute-hms-test.sh | 207 --
packaging/src/main/assembly/src.xml | 1 -
pom.xml | 1 -
testutils/metastore/execute-test-on-lxc.sh | 161 --
testutils/ptest/Buffer.py | 36 -
testutils/ptest/Process.py | 40 -
testutils/ptest/README | 66 -
testutils/ptest/Report.py | 230 --
testutils/ptest/Ssh.py | 126 -
testutils/ptest/config.py | 84 -
testutils/ptest/hivetest.py | 607 -----
testutils/ptest/templates/Properties.html | 41 -
testutils/ptest/templates/TestCase.html | 43 -
testutils/ptest/templates/TestRun.html | 90 -
testutils/ptest/templates/TestSuite.html | 37 -
testutils/ptest/templates/common.css | 72 -
testutils/ptest2/.gitignore | 11 -
testutils/ptest2/README.md | 100 -
testutils/ptest2/conf/cloudhost.properties.example | 54 -
.../ptest2/conf/deployed/master-mr2.properties | 222 --
.../ptest2/conf/example-apache-trunk.properties | 87 -
.../ptest2/conf/server-env.properties.example | 23 -
testutils/ptest2/pom.xml | 296 ---
.../java/org/apache/hive/ptest/api/Status.java | 111 -
.../apache/hive/ptest/api/client/PTestClient.java | 338 ---
.../hive/ptest/api/request/TestListRequest.java | 35 -
.../hive/ptest/api/request/TestLogRequest.java | 53 -
.../hive/ptest/api/request/TestStartRequest.java | 76 -
.../hive/ptest/api/request/TestStatusRequest.java | 40 -
.../hive/ptest/api/request/TestStopRequest.java | 40 -
.../hive/ptest/api/response/GenericResponse.java | 25 -
.../hive/ptest/api/response/TestListResponse.java | 53 -
.../hive/ptest/api/response/TestLogResponse.java | 59 -
.../hive/ptest/api/response/TestStartResponse.java | 44 -
.../apache/hive/ptest/api/response/TestStatus.java | 82 -
.../ptest/api/response/TestStatusResponse.java | 50 -
.../hive/ptest/api/response/TestStopResponse.java | 39 -
.../hive/ptest/api/server/ExecutionController.java | 241 --
.../org/apache/hive/ptest/api/server/Test.java | 117 -
.../apache/hive/ptest/api/server/TestExecutor.java | 193 --
.../apache/hive/ptest/api/server/TestLogger.java | 264 --
.../hive/ptest/execution/AbortDroneException.java | 29 -
.../org/apache/hive/ptest/execution/Constants.java | 27 -
.../java/org/apache/hive/ptest/execution/Dirs.java | 41 -
.../org/apache/hive/ptest/execution/Drone.java | 68 -
.../hive/ptest/execution/ExecutionPhase.java | 202 --
.../apache/hive/ptest/execution/HostExecutor.java | 469 ----
.../hive/ptest/execution/HostExecutorBuilder.java | 25 -
.../apache/hive/ptest/execution/JIRAService.java | 464 ----
.../hive/ptest/execution/JUnitReportParser.java | 155 --
.../apache/hive/ptest/execution/LocalCommand.java | 154 --
.../hive/ptest/execution/LocalCommandFactory.java | 36 -
.../hive/ptest/execution/LogDirectoryCleaner.java | 99 -
.../org/apache/hive/ptest/execution/PTest.java | 411 ----
.../org/apache/hive/ptest/execution/Phase.java | 207 --
.../org/apache/hive/ptest/execution/PrepPhase.java | 75 -
.../hive/ptest/execution/ReportingPhase.java | 43 -
.../org/apache/hive/ptest/execution/Templates.java | 64 -
.../hive/ptest/execution/TestCheckPhase.java | 93 -
.../hive/ptest/execution/TestsFailedException.java | 27 -
.../apache/hive/ptest/execution/YetusPhase.java | 150 --
.../apache/hive/ptest/execution/conf/Context.java | 345 ---
.../conf/ExecutionContextConfiguration.java | 104 -
.../ptest/execution/conf/FileListProvider.java | 31 -
.../org/apache/hive/ptest/execution/conf/Host.java | 95 -
.../hive/ptest/execution/conf/QFileTestBatch.java | 148 --
.../hive/ptest/execution/conf/TestBatch.java | 49 -
.../ptest/execution/conf/TestConfiguration.java | 306 ---
.../hive/ptest/execution/conf/TestParser.java | 287 ---
.../hive/ptest/execution/conf/UnitTestBatch.java | 122 -
.../execution/conf/UnitTestPropertiesParser.java | 666 -----
.../execution/context/CloudComputeService.java | 282 ---
.../context/CloudExecutionContextProvider.java | 531 ----
.../context/CreateHostsFailedException.java | 30 -
.../ptest/execution/context/ExecutionContext.java | 73 -
.../context/ExecutionContextProvider.java | 38 -
.../context/FixedExecutionContextProvider.java | 90 -
.../context/ServiceNotAvailableException.java | 30 -
.../ptest/execution/ssh/AbstractSSHCommand.java | 70 -
.../ptest/execution/ssh/AbstractSSHResult.java | 62 -
.../execution/ssh/NonZeroExitCodeException.java | 29 -
.../hive/ptest/execution/ssh/RSyncCommand.java | 74 -
.../ptest/execution/ssh/RSyncCommandExecutor.java | 114 -
.../hive/ptest/execution/ssh/RSyncResult.java | 47 -
.../ptest/execution/ssh/RemoteCommandResult.java | 28 -
.../hive/ptest/execution/ssh/SSHCommand.java | 55 -
.../ptest/execution/ssh/SSHCommandExecutor.java | 88 -
.../ptest/execution/ssh/SSHExecutionException.java | 36 -
.../apache/hive/ptest/execution/ssh/SSHResult.java | 39 -
testutils/ptest2/src/main/resources/batch-exec.vm | 108 -
.../ptest2/src/main/resources/log4j2.properties | 77 -
.../ptest2/src/main/resources/smart-apply-patch.sh | 101 -
testutils/ptest2/src/main/resources/source-prep.vm | 133 -
testutils/ptest2/src/main/resources/yetus-exec.vm | 56 -
.../WEB-INF/config/web-application-config.xml | 29 -
.../main/webapp/WEB-INF/config/webmvc-config.xml | 29 -
testutils/ptest2/src/main/webapp/WEB-INF/web.xml | 55 -
.../hive/ptest/api/server/TestTestExecutor.java | 142 --
.../hive/ptest/api/server/TestTestLogger.java | 54 -
.../hive/ptest/execution/AbstractTestPhase.java | 140 --
.../hive/ptest/execution/ExtendedAssert.java | 54 -
.../ptest/execution/MockLocalCommandFactory.java | 47 -
.../ptest/execution/MockRSyncCommandExecutor.java | 81 -
.../ptest/execution/MockSSHCommandExecutor.java | 74 -
.../TestCleanupPhase.testExecute.approved.txt | 0
.../hive/ptest/execution/TestExecutionPhase.java | 190 --
...estExecutionPhase.testFailingQFile.approved.txt | 14 -
...ExecutionPhase.testFailingUnitTest.approved.txt | 14 -
...xecutionPhase.testPassingQFileTest.approved.txt | 14 -
...ExecutionPhase.testPassingUnitTest.approved.txt | 14 -
...xecutionPhase.testTimedOutUnitTest.approved.txt | 14 -
.../hive/ptest/execution/TestHostExecutor.java | 256 --
.../TestHostExecutor.testBasic.approved.txt | 12 -
...stExecutor.testIsolatedFailsOnExec.approved.txt | 5 -
...ecutor.testIsolatedFailsOnRsyncOne.approved.txt | 4 -
...or.testIsolatedFailsOnRsyncUnknown.approved.txt | 4 -
...stExecutor.testParallelFailsOnExec.approved.txt | 5 -
...tExecutor.testParallelFailsOnRsync.approved.txt | 4 -
...ostExecutor.testShutdownBeforeExec.approved.txt | 1 -
.../hive/ptest/execution/TestJIRAService.java | 167 --
...tJIRAService.testErrorWithMessages.approved.txt | 20 -
...RAService.testErrorWithoutMessages.approved.txt | 14 -
.../TestJIRAService.testFailAdd.approved.txt | 21 -
.../TestJIRAService.testFailNoAdd.approved.txt | 21 -
.../TestJIRAService.testSuccessAdd.approved.txt | 16 -
.../TestJIRAService.testSuccessNoAdd.approved.txt | 16 -
.../hive/ptest/execution/TestLocalCommand.java | 51 -
.../ptest/execution/TestLogDirectoryCleaner.java | 58 -
.../org/apache/hive/ptest/execution/TestPhase.java | 127 -
...TestPhase.testExecHostsWithFailure.approved.txt | 2 -
...Phase.testExecInstancesWithFailure.approved.txt | 2 -
...calToRemoteInstancesWithFailureOne.approved.txt | 3 -
...oRemoteInstancesWithFailureUnknown.approved.txt | 3 -
.../apache/hive/ptest/execution/TestPrepPhase.java | 52 -
.../TestPrepPhase.testExecute.approved.txt | 5 -
.../hive/ptest/execution/TestReportParser.java | 88 -
.../hive/ptest/execution/TestReportingPhase.java | 43 -
.../TestReportingPhase.testExecute.approved.txt | 3 -
.../apache/hive/ptest/execution/TestScripts.java | 215 --
...TestScripts.testAlternativeTestJVM.approved.txt | 107 -
.../execution/TestScripts.testBatch.approved.txt | 107 -
.../execution/TestScripts.testPrepGit.approved.txt | 107 -
.../TestScripts.testPrepHadoop1.approved.txt | 119 -
.../TestScripts.testPrepNone.approved.txt | 107 -
.../execution/TestScripts.testPrepSvn.approved.txt | 107 -
.../hive/ptest/execution/TestTestCheckPhase.java | 117 -
.../hive/ptest/execution/conf/TestContext.java | 68 -
.../ptest/execution/conf/TestQFileTestBatch.java | 71 -
.../execution/conf/TestTestConfiguration.java | 107 -
.../hive/ptest/execution/conf/TestTestParser.java | 160 --
.../conf/TestUnitTestPropertiesParser.java | 672 ------
.../execution/context/TestCloudComputeService.java | 72 -
.../context/TestCloudExecutionContextProvider.java | 119 -
.../execution/ssh/TestRSyncCommandExecutor.java | 87 -
.../execution/ssh/TestSSHCommandExecutor.java | 89 -
.../ptest2/src/test/resources/HIVE-10761.6.patch | 2539 --------------------
.../ptest2/src/test/resources/HIVE-11271.4.patch | 606 -----
.../ptest2/src/test/resources/HIVE-19077.1.patch | 14 -
.../ptest2/src/test/resources/HIVE-9377.1.patch | 25 -
.../ptest2/src/test/resources/SomeTest-failure.xml | 7 -
.../ptest2/src/test/resources/SomeTest-success.xml | 9 -
.../ptest2/src/test/resources/log4j2.properties | 62 -
.../ptest2/src/test/resources/remove-test.patch | 33 -
.../test/resources/test-configuration.properties | 36 -
.../test/resources/test-configuration2.properties | 150 --
.../resources/test-outputs/SomeTest-truncated.xml | 4 -
...ST-org.apache.hadoop.hive.cli.TestCliDriver.xml | 34 -
...x-ba31fb54-1d7f-4c70-a89d-477b7d155191-hive.log | 0
.../resources/test-outputs/index_auth.q-hive.log | 10 -
...ST-org.apache.hadoop.hive.cli.TestCliDriver.xml | 53 -
...7-1b5c-45ed-ba29-14450f27db8b-TestCliDriver.txt | 0
...q-ab8536a7-1b5c-45ed-ba29-14450f27db8b-hive.log | 0
...6-d2b0-4e13-9761-11f750627ad1-TestCliDriver.txt | 0
...q-6fa31776-d2b0-4e13-9761-11f750627ad1-hive.log | 0
...ST-org.apache.hadoop.hive.cli.TestCliDriver.xml | 54 -
.../skewjoin_union_remove_1.q-TestCliDriver.txt | 10 -
...ST-org.apache.hadoop.hive.cli.TestCliDriver.xml | 53 -
...f-1b9c-4874-924c-b2107ca7b07c-TestCliDriver.txt | 0
...q-acb9de8f-1b9c-4874-924c-b2107ca7b07c-hive.log | 0
180 files changed, 19920 deletions(-)
diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh
deleted file mode 100644
index f660fcb..0000000
--- a/dev-support/jenkins-execute-build.sh
+++ /dev/null
@@ -1,144 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-set -x
-
-. jenkins-common.sh
-
-# Build the ptest framework locally
-build_ptest_client() {
- [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
-
- test -d $PTEST_BUILD_DIR || mkdir -p $PTEST_BUILD_DIR
- cd $PTEST_BUILD_DIR && rm -rf hive
-
- unset GIT_CLONE_ARGS
- if [ -n "${PTEST_GIT_BRANCH}" ]; then
- GIT_CLONE_ARGS=" -b ${PTEST_GIT_BRANCH}"
- fi
- if [ -z "${PTEST_GIT_REPO}" ]; then
- PTEST_GIT_REPO=https://github.com/apache/hive.git
- fi
- GIT_CLONE_ARGS=${GIT_CLONE_ARGS}" ${PTEST_GIT_REPO} hive"
-
- git clone --depth 1 ${GIT_CLONE_ARGS}
- cd hive/testutils/ptest2
- mvn clean package -B -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$MVN_REPO_LOCAL
-}
-
-# Call the ptest server to run all tests
-call_ptest_server() {
- [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
-
- read -s -p "JIRA password: " JIRA_PASSWORD
-
- build_ptest_client || return 1
-
- local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-3.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*"
-
- java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \
- --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@"
-}
-
-# Unpack all test results
-unpack_test_results() {
- [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
-
- cd "$PTEST_BUILD_DIR/hive/testutils/ptest2/target"
- if [[ -f test-results.tar.gz ]]; then
- rm -rf $PTEST_BUILD_DIR/test-results/
- tar zxf test-results.tar.gz -C $PTEST_BUILD_DIR
- fi
-}
-
-# Check required arguments
-test -n "$TEST_HANDLE" || fail "TEST_HANDLE must be specified and cannot be empty."
-test -n "$PTEST_API_ENDPOINT" || fail "PTEST_API_ENDPOINT must be specified and cannot be empty."
-test -n "$PTEST_LOG_ENDPOINT" || fail "PTEST_LOG_ENDPOINT must be specified and cannot be empty."
-
-# WORKSPACE is an environment variable created by Jenkins, and it is the directory where the build is executed.
-# If not set, then default to $HOME
-MVN_REPO_LOCAL=${WORKSPACE:-$HOME}/.m2/repository
-
-# Directory where to build the ptest framework
-PTEST_BUILD_DIR="$PWD/hive/build"
-
-# Default profile in case a patch does not have a profile assigned, or
-# if this script is executed without JIRA_ISSUE defined
-DEFAULT_BUILD_PROFILE="master-mr2"
-
-# Optional parameters that might be passed to the ptest client command
-optionalArgs=()
-
-if [ -n "$JIRA_ISSUE" ]; then
- JIRA_INFO_FILE=`mktemp`
- trap "rm -f $JIRA_INFO_FILE" EXIT
-
- initialize_jira_info $JIRA_ISSUE $JIRA_INFO_FILE || fail "Error: Cannot initialize JIRA information."
-
- if ! is_patch_available $JIRA_INFO_FILE; then
- fail "$JIRA_ISSUE is not 'Patch Available'."
- fi
-
- if is_check_no_precommit_tests_set $JIRA_INFO_FILE; then
- fail "$JIRA_ISSUE has tag NO PRECOMMIT TESTS"
- fi
-
- JIRA_PATCH_URL=`get_jira_patch_url $JIRA_INFO_FILE`
- if [ -z "$JIRA_PATCH_URL" ]; then
- fail "Unable to find attachment for $JIRA_ISSUE"
- fi
-
- attachment_id=`get_attachment_id $JIRA_PATCH_URL`
- if is_patch_already_tested "$attachment_id" "$TEST_HANDLE" "$JIRA_INFO_FILE"; then
- fail "attachment $attachment_id is already tested for $JIRA_ISSUE"
- fi
-
- # Use the BUILD_PROFILE if it is provided.
- if [ -z ${BUILD_PROFILE} ]; then
- BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE`
- if [ -z "$BUILD_PROFILE" ]; then
- BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
- fi
- fi
-
- if is_clear_cache_set $JIRA_INFO_FILE; then
- optionalArgs+=(--clearLibraryCache)
- fi
-
- optionalArgs+=(--patch "${JIRA_ROOT_URL}${JIRA_PATCH_URL}" --jira "$JIRA_ISSUE")
-
- echo "ISSUE: $JIRA_ISSUE PROFILE: $BUILD_PROFILE"
-else
- # If not JIRA is specified, and no BUILD_PROFILE provided, then use a default profile
- if [ -z ${BUILD_PROFILE} ]; then
- BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
- fi
-
- echo "ISSUE: unspecified PROFILE: $BUILD_PROFILE"
-fi
-
-set +e
-
-call_ptest_server --testHandle "$TEST_HANDLE" --endpoint "$PTEST_API_ENDPOINT" --logsEndpoint "$PTEST_LOG_ENDPOINT" \
- --profile "$BUILD_PROFILE" ${optionalArgs[@]} "$@"
-
-ret=$?
-
-unpack_test_results
-
-exit $ret
diff --git a/dev-support/jenkins-execute-hms-test.sh b/dev-support/jenkins-execute-hms-test.sh
deleted file mode 100644
index ac36c80..0000000
--- a/dev-support/jenkins-execute-hms-test.sh
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-# This script is run on jenkins only, and it creates some LXC containers
-# in order to execute the metastore-upgrade-tests for different
-# server configurations.
-
-set -x
-. jenkins-common.sh
-
-test -n "$BRANCH" || fail "BRANCH must be specified"
-test -n "$ISSUE_NUM" || fail "ISSUE_NUM must be specified"
-test -n "$SSH_HOST" || fail "SSH_HOST must be specified"
-test -n "$SSH_KEY" || fail "SSH_KEY must be specified"
-test -n "$BUILD_TAG" || fail "BUILD_TAG must be specified"
-
-test -n "$WORKSPACE" || WORKSPACE="$HOME"
-
-export JIRA_NAME="HIVE-${ISSUE_NUM}"
-export ROOT=$PWD
-export JIRA_ROOT_URL="https://issues.apache.org"
-export BUILD_TAG="${BUILD_TAG##jenkins-}"
-
-process_jira
-
-# Jenkins may call this script with BRANCH=trunk to refer to SVN.
-# We use git here, so we change to master.
-[[ "$BRANCH" = "trunk" ]] && BRANCH="master"
-
-PUBLISH_VARS=(
- BUILD_STATUS=buildStatus
- BUILD_TAG=buildTag
- LOGS_URL=logsURL
- JENKINS_URL=jenkinsURL
- PATCH_URL=patchUrl
- JIRA_NAME=jiraName
- JIRA_URL=jiraUrl
- BRANCH=branch
- REPO=repository
- REPO_NAME=repositoryName
- REPO_TYPE=repositoryType
- TESTS_EXECUTED=numTestsExecuted
- FAILED_TESTS=failedTests
- MESSAGES=messages
-)
-
-BUILD_STATUS=0
-PATCH_URL="${JIRA_ROOT_URL}${PATCH_URL}"
-TESTS_EXECUTED=0
-FAILED_TESTS=()
-MESSAGES=()
-
-PROFILE_PROPERTIES_FILE="/usr/local/hiveptest/etc/public/${BUILD_PROFILE}.properties"
-[[ -f $PROFILE_PROPERTIES_FILE ]] || fail "$PROFILE_PROPERTIES_FILE file does not exist."
-
-profile_properties_get() {
- grep "^$1 = " "$PROFILE_PROPERTIES_FILE" | awk '{print $3}'
-}
-
-REPO_TYPE=$(profile_properties_get "repositoryType")
-REPO_NAME=$(profile_properties_get "repository")
-REPO=$(profile_properties_get "repositoryName")
-JIRA_URL=$(profile_properties_get "jiraUrl")
-
-# Avoid showing this information on Jira
-set +x
-JIRA_USER=$(profile_properties_get "jiraUser")
-JIRA_PASS=$(profile_properties_get "jiraPassword")
-set -x
-
-JENKINS_URL=$(profile_properties_get "jenkinsURL")
-LOGS_URL=$(profile_properties_get "logsURL")
-
-build_ptest2() {
- local path="$1"
- local curpath="$PWD"
-
- test -d $path || mkdir -p $path
- rm -rf $path
- git clone --depth 1 -b $BRANCH https://github.com/apache/hive.git $path/ || return 1
- cd $path/testutils/ptest2
- mvn clean package -B -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2 || return 1
-
- cd $curpath
-}
-
-publish_results() {
- local file="$1"
-
- build_ptest2 "hive/build" || return 1
-
- # Avoid showing this information on Jira
- set +x
- java -cp "hive/build/testutils/ptest2/target/*:hive/build/testutils/ptest2/target/lib/*" org.apache.hive.ptest.execution.JIRAService \
- --user "$JIRA_USER" \
- --password "$JIRA_PASS" \
- --file "$file"
- set -x
-}
-
-add_json_object() {
- echo -n "\"$2\" : \"$3\"" >> $1
-}
-
-add_json_array() {
- echo -n "\"$2\" : [ $3 ]" >> $1
-}
-
-create_publish_file() {
- local json_file=$(mktemp)
- local arr_length=${#PUBLISH_VARS[@]}
-
- vars_added=0
- echo "{" > $json_file
- for i in ${PUBLISH_VARS[@]}
- do
- var=${i%=*}
- key=${i#*=}
- val=
-
- # Treat this as an array
- if [[ $var = "FAILED_TESTS" ]]; then
- if [[ ${#FAILED_TESTS[@]} -gt 0 ]]; then
- val=$(printf "\"%s\"," "${FAILED_TESTS[@]}")
- val=${val%?}
- fi
- add_json_array $json_file $key "$val"
- elif [[ $var = "MESSAGES" ]]; then
- if [[ ${#MESSAGES[@]} -gt 0 ]]; then
- val=$(printf "\"%s\"," "${MESSAGES[@]}")
- val=${val%?}
- fi
- add_json_array $json_file $key "$val"
- else
- val=${!var}
- add_json_object $json_file $key $val
- fi
-
- vars_added=$((vars_added+1))
-
- if [[ $vars_added -lt $arr_length ]]; then
- echo "," >> $json_file
- else
- echo >> $json_file
- fi
- done
- echo "}" >> $json_file
-
- echo $json_file
-}
-
-if patch_contains_hms_upgrade "$PATCH_URL"; then
- ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SSH_KEY $SSH_HOST "
- rm -rf hive/ &&
- git clone --depth 1 -b $BRANCH https://github.com/apache/hive.git &&
- cd hive/ &&
- curl ${PATCH_URL} | bash -x testutils/ptest2/src/main/resources/smart-apply-patch.sh - &&
- sudo bash -x testutils/metastore/execute-test-on-lxc.sh --patch \"${PATCH_URL}\" --branch $BRANCH
- "
- BUILD_STATUS=$?
- if [[ $BUILD_STATUS = 0 ]]; then
- tmp_test_log="/tmp/execute-test-on-lxc.sh.log"
-
- scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SSH_KEY ${SSH_HOST}:"$tmp_test_log" "$tmp_test_log" || exit 1
- TESTS_EXECUTED=$(cat "$tmp_test_log" | grep "Executing sql test" | wc -l)
-
- while read line
- do
- if echo $line | grep 'Test failed' > /dev/null; then
- FAILED_TESTS+=("$line")
- elif echo $line | grep 'Executing sql test' >/dev/null; then
- # Remove 'Executing sql test' line from MESSAGES log to avoid a verbose
- # comment on JIRA
- continue
- fi
-
- MESSAGES+=("$line")
- done < "$tmp_test_log"
-
- rm "$tmp_test_log"
- fi
-
- json_file=$(create_publish_file)
- publish_results "$json_file"
- ret=$?
-
- rm "$json_file"
- exit $ret
-fi
diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml
index 65ed0c8..625fcf2 100644
--- a/packaging/src/main/assembly/src.xml
+++ b/packaging/src/main/assembly/src.xml
@@ -41,7 +41,6 @@
<exclude>**/.project</exclude>
<exclude>**/.settings/**</exclude>
<exclude>**/thirdparty/**</exclude>
- <exclude>testutils/ptest2/velocity.log*</exclude>
</excludes>
<includes>
diff --git a/pom.xml b/pom.xml
index 712213a..47c5d53 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1672,7 +1672,6 @@
<exclude>**/*.m</exclude>
<exclude>**/gen-java/**</exclude>
<exclude>**/testdata/**</exclude>
- <exclude>**/ptest2/*.md</exclude>
<exclude>**/test/org/apache/hadoop/hive/hbase/avro/**</exclude>
<exclude>**/avro_test.avpr</exclude>
<exclude>**/xmlReport.pl</exclude>
diff --git a/testutils/metastore/execute-test-on-lxc.sh b/testutils/metastore/execute-test-on-lxc.sh
deleted file mode 100644
index b32ee9d..0000000
--- a/testutils/metastore/execute-test-on-lxc.sh
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-# This script is run on jenkins only, and it creates some LXC containers
-# in order to execute the metastore-upgrade-tests for different
-# server configurations.
-
-cd $(dirname $0)
-
-OUT_LOG="/tmp/$(basename $0).log"
-rm -f $OUT_LOG
-
-log() {
- echo "$@"
- echo "$@" >> $OUT_LOG
-}
-
-
-fail() {
- echo $@
- exit 1
-}
-
-[[ $# != 4 ]] && fail "Usage: $0 --patch PATH_URL --branch BRANCH"
-
-PATCH_URL=
-BRANCH=
-while [[ $# -gt 0 ]]; do
- if [[ $1 = "--patch" ]]; then
- PATCH_URL=$2
- elif [[ $1 = "--branch" ]]; then
- BRANCH=$2
- fi
-
- shift 2
-done
-
-test -n "$BRANCH" || fail "--branch value is required."
-test -n "$PATCH_URL" || fail "--patch value is required."
-
-get_supported_dbs() {
- ls dbs/ -1
-}
-
-lxc_get_ip() {
- lxc-ls -f "^$1$" | tail -1 | awk '{print $3}' | tr -d ,
-}
-
-lxc_exists() {
- lxc-ls "^$1$" | grep $1 >/dev/null
-}
-
-lxc_create() {
- # Oracle works better on 32-bits machines only, so we use i386 containers.
- lxc-create -n $1 -t download -- --dist "ubuntu" --release "trusty" --arch "i386" || return 1
- lxc_start $1 || return 1
-}
-
-lxc_running() {
- lxc-ls -f "^$1$" | tail -1 | awk '{print $2}' | grep "RUNNING" >/dev/null
-}
-
-lxc_start() {
- lxc-start -n $1 --daemon || return 1
- lxc-wait -n $1 -s RUNNING || return 1
- sleep 10 # wait a little longer
-}
-
-lxc_stop() {
- lxc-stop -n $1
-}
-
-lxc_prepare() {
- echo "Downloading hive source code from SVN, branch='$BRANCH' ..."
-
- lxc-attach -n $1 -- apt-get update
- lxc-attach -n $1 -- apt-get install -y patch git wget curl
-
- tmpfile=$(mktemp)
- cat>$tmpfile<<EOF
-rm -rf hive
-mkdir hive
-git clone --depth 1 -b $BRANCH https://github.com/apache/hive.git >/dev/null
-cd hive
-wget $PATCH_URL -O hms.patch
-bash -x testutils/ptest2/src/main/resources/smart-apply-patch.sh hms.patch
-EOF
-
- lxc-attach -n $1 -- bash -x -e < $tmpfile
-}
-
-lxc_print_metastore_log() {
- lxc-attach -n $1 -- cat /tmp/metastore-upgrade-test.sh.log
-}
-
-run_tests() {
- lxc-attach -n $1 -- bash hive/testutils/metastore/metastore-upgrade-test.sh --db $1
-}
-
-# Install LXC packages if needed
-if ! which lxc-create >/dev/null; then
- apt-get update
- apt-get -y install lxc || exit 1
-fi
-
-for d in $(get_supported_dbs)
-do
- name=$(basename $d)
-
- # Create container
- if ! lxc_exists $name; then
- log "LXC $name is not found. Creating new container..."
- lxc_create $name || exit 1
- log "Container created."
- else
- log "LXC $name found."
- fi
-
- # Start container
- if ! lxc_running $name; then
- log "LXC $name is not started. Starting container..."
- lxc_start $name || exit 1
- log "Container started."
- fi
-
- # Prepare container
- log "Preparing $name container..."
- lxc_prepare $name || exit 1
- log "Container prepared."
-
- # Execute metastore upgrade tests
- echo "Running metastore upgrade tests for $name..."
- run_tests $name
- rc=$?
-
- log "$(lxc_print_metastore_log $name)"
- lxc_stop $name
-
- if [[ $rc != 0 ]]; then
- log "Tests failed. Exiting with error code (1)."
- exit 1
- fi
-done
diff --git a/testutils/ptest/Buffer.py b/testutils/ptest/Buffer.py
deleted file mode 100644
index 0ba4be8..0000000
--- a/testutils/ptest/Buffer.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import collections
-
-class Buffer():
- def __init__(self, size = 100, abandon_output = True):
- self.buf_short = collections.deque(maxlen = size)
- self.buf = []
- self.abandon_output = abandon_output
-
- def put(self, line):
- if not self.abandon_output:
- self.buf.append(line)
- self.buf_short.append(line)
-
- def get_short(self):
- return ''.join(self.buf_short)
-
- def get_long(self):
- if self.abandon_output:
- return None
- else:
- return ''.join(self.buf)
diff --git a/testutils/ptest/Process.py b/testutils/ptest/Process.py
deleted file mode 100644
index 77671f6..0000000
--- a/testutils/ptest/Process.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-from subprocess import Popen, PIPE, STDOUT
-from shlex import split
-
-from Buffer import Buffer
-
-def run(cmd, quiet = False, abandon_output = True):
- proc = Popen(split(cmd), stdout = PIPE, stderr = STDOUT)
- buf = Buffer(abandon_output = abandon_output)
- line = proc.stdout.readline()
- while len(line):
- buf.put(line)
- if not quiet:
- print(line, end = '')
- line = proc.stdout.readline()
- # Process could probably close the descriptor before exiting.
- proc.wait()
- if proc.returncode != 0:
- raise Exception('Process exited with a non-zero return code. ' +
- 'Last output of the program:\n\n' +
- '---------- Start of exception log --\n' +
- buf.get_short().strip() +
- '\n---------- End of exception log --\n')
- return buf.get_long()
diff --git a/testutils/ptest/README b/testutils/ptest/README
deleted file mode 100644
index 0f071fd..0000000
--- a/testutils/ptest/README
+++ /dev/null
@@ -1,66 +0,0 @@
-= Configuration file =
-
-Configuration file is JSON formated, example:
-
-{
- "qfile_hosts": [
- ["username@hostname1", 4],
- ["username@hostname2", 8],
- ["username@hostname3", 8]
- ],
-
- "other_hosts": [
- ["username@hostname4", 5],
- ["username@hostname5", 8]
- ],
-
- "master_base_path": "${{HOME}}/hivetests",
- "host_base_path": "/data/users/hivetests",
- "java_home": "/opt/jdk"
-}
-
-== qfile_hosts ==
-List of hosts that should run TestCliDriver and TestNegativeCliDriver test
-cases. Number following the host name is number of simultaneous tests that
-should be run on this host, you should probably set it near number of cores that
-host has.
-
-== other_hosts ==
-List of hosts that should run all other test cases. Number has the same meaning
-as in `qfile_hosts`.
-
-== master_base_path ==
-Path on localhost (master) where this script can build Hive, store reports, etc.
-This path should be available from every slave node and should point to the same
-data (home on NFS would be a good choice). If you specify `HIVE_PTEST_SUFFIX`
-environmental variable the actual path used will be
-`master_base_path-HIVE_PTEST_SUFFIX`.
-
-== host_base_path ==
-Path on slaves where Hive repo will be cloned and tests will be run.
-'-your_user_name' will be actually appended to this path to allow parallel runs
-by different users. `HIVE_PTEST_SUFFIX` affects this path the same as it
-affects `master_base_path`, and will be appended if needed.
-
-== java_home ==
-Should point to Java environment that should be used.
-
-== About paths ==
-You can use environmental variables with `${{my_env}}`, as home is used in the
-example.
-
-You shouldn't point this paths to your work repository or any directory that
-stores data you don't want to lose. This script might wipe everything under
-`master_base_path` and `host_base_path` as needed.
-
-== Example paths ==
-Assuming your user name is 'foo' and you are using example config defined above
-this will be final paths used by the script.
-
-unset HIVE_PTEST_SUFFIX
-master_base_path = /home/foo/hivetests
-host_base_path = /mnt/drive/hivetests-foo
-
-export HIVE_PTEST_SUFFIX=second_run
-master_base_path = /home/foo/hivetests-second_run
-host_base_path = /mnt/drive/hivetests-foo-second_run
diff --git a/testutils/ptest/Report.py b/testutils/ptest/Report.py
deleted file mode 100755
index ddbd87e..0000000
--- a/testutils/ptest/Report.py
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import os
-import os.path
-import re
-import base64
-import argparse
-from xml.dom import Node
-from xml.dom.minidom import parseString
-
-from mako.template import Template
-
-report_dir = os.path.dirname(os.path.realpath(__file__))
-one_file = False
-
-class TemplateRenderer():
- def __init__(self, template):
- self.__template = template
-
- def render(self, file_name = None):
- if file_name is None:
- file_name = self.__template
- if not file_name.startswith('/'):
- current_dir = os.path.dirname(os.path.realpath(__file__))
- file_name = os.path.join(current_dir, file_name)
- return Template(filename = file_name).render(this = self)
-
- def render_link(self, link_name, file_name):
- if one_file:
- return 'data:text/html;charset=utf-8;base64,' + \
- base64.b64encode(self.render(file_name))
- else:
- part = self.render(file_name)
- with open(report_dir + '/' + link_name, 'w') as f:
- f.write(part)
- return link_name
-
- def render_files(self):
- report = self.render()
- with open(report_dir + '/report.html', 'w') as f:
- f.write(report)
-
-class TestCase(TemplateRenderer):
- def __init__(self, element):
- TemplateRenderer.__init__(self, 'templates/TestCase.html')
-
- self.__class_name = element.getAttribute('classname')
- self.__name = element.getAttribute('name')
- self.__time = float(element.getAttribute('time'))
- self.__failure = False
- self.__error = False
- self.__log = None
-
- for child in element.childNodes:
- if child.nodeType == Node.ELEMENT_NODE and child.tagName == 'failure':
- self.__failure = True
- self.__log = child.firstChild.nodeValue
- elif child.nodeType == Node.ELEMENT_NODE and child.tagName == 'error':
- self.__error = True
- self.__log = child.firstChild.nodeValue
-
- def success(self):
- return not (self.failure() or self.error())
-
- def failure(self):
- return self.__failure
-
- def error(self):
- return self.__error
-
- def get_log(self):
- return self.__log
-
- def get_name(self):
- return self.__name
-
- def get_time(self):
- return self.__time
-
-class TestSuite(TemplateRenderer):
- def __init__(self, text):
- TemplateRenderer.__init__(self, 'templates/TestSuite.html')
-
- self.properties = {}
- self.test_cases = []
-
- xml = parseString(text)
- self.__populate_properties(xml)
- self.__populate_test_cases(xml)
-
- top = xml.getElementsByTagName('testsuite')[0]
- self.__errors = int(top.getAttribute('errors'))
- self.__failures = int(top.getAttribute('failures'))
- self.__tests = int(top.getAttribute('tests'))
- self.__host_name = top.getAttribute('hostname')
- dist_dir = self.properties['dist.dir']
- build_number = re.findall(self.__host_name + '-([0-9]+)$', dist_dir)
- if build_number:
- # Multiple builds per host.
- self.__host_name += '-' + build_number[0]
- self.__name = top.getAttribute('name').split('.')[-1]
- self.__time = float(top.getAttribute('time'))
-
- def __populate_properties(self, xml):
- properties = xml.getElementsByTagName('property')
- for prop in properties:
- self.properties[prop.getAttribute('name')] = prop.getAttribute('value')
-
- def __populate_test_cases(self, xml):
- test_cases = xml.getElementsByTagName('testcase')
- for test in test_cases:
- self.test_cases.append(TestCase(test))
-
- def tests(self):
- return self.__tests
-
- def failures(self):
- return self.__failures
-
- def errors(self):
- return self.__errors
-
- def passes(self):
- return self.tests() - self.failures() - self.errors()
-
- def time(self):
- return self.__time
-
- def host_name(self):
- return self.__host_name
-
- def name(self):
- return self.__name
-
- def label(self):
- return self.host_name() + '-' + self.name()
-
-class TestRun(TemplateRenderer):
- def __init__(self, pwd):
- TemplateRenderer.__init__(self, 'templates/TestRun.html')
-
- self.test_suites = []
-
- files = os.listdir(pwd)
- pattern = re.compile('^TEST-.*\.xml$')
- for f in files:
- if pattern.search(f) is not None:
- with open(os.path.join(pwd, f)) as handle:
- self.test_suites.append(TestSuite(handle.read()))
-
- def passes(self):
- return reduce(lambda acc, x: acc + x.passes(), self.test_suites, 0)
-
- def failures(self):
- return reduce(lambda acc, x: acc + x.failures(), self.test_suites, 0)
-
- def errors(self):
- return reduce(lambda acc, x: acc + x.errors(), self.test_suites, 0)
-
- def tests(self):
- return reduce(lambda acc, x: acc + x.tests(), self.test_suites, 0)
-
- def time(self):
- return reduce(lambda acc, x: acc + x.time(), self.test_suites, 0)
-
- def success_rate(self):
- if self.tests():
- return 100.0 * self.passes() / self.tests()
- else:
- return 100.0
-
-def make_report(args):
- global report_dir, one_file
- report_dir = args.report_dir
- one_file = args.one_file
-
- test_run = TestRun(args.log_dir)
- test_run.render_files()
- print('Summary:')
- print(' tests run:', test_run.tests())
- print(' failures:', test_run.failures())
- print(' errors:', test_run.errors())
-
- failed_results = {}
- files = os.listdir(args.log_dir)
- pattern = re.compile('^([^-]+-[^-]+)-(.*)\.fail$')
- for f in files:
- match = pattern.findall(f)
- if match:
- (host, test, ) = match[0]
- if host not in failed_results:
- failed_results[host] = []
- failed_results[host].append(test)
- if failed_results:
- print()
- print('Some tests faled to produce a log and are not included in the report:')
- for host in failed_results:
- print(' {0}:'.format(host))
- for test in failed_results[host]:
- print(' {0}'.format(test))
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description = 'Create HTML report from JUnit logs.')
- parser.add_argument(dest = 'log_dir',
- help = 'Path to directory containing JUnit logs')
- parser.add_argument(dest = 'report_dir',
- help = 'Where should the report be generated')
- parser.add_argument('--one-file', action = 'store_true', dest = 'one_file',
- help = 'Inline everything and generate only one file')
- args = parser.parse_args()
-
- make_report(args)
diff --git a/testutils/ptest/Ssh.py b/testutils/ptest/Ssh.py
deleted file mode 100644
index 9f4456d..0000000
--- a/testutils/ptest/Ssh.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from threading import Thread
-from Queue import Queue
-
-import Process
-
-class SSHConnection():
- def __init__(self, host, num = None):
- self.host = host
- if num is None:
- self.hostname = host
- else:
- self.hostname = host + '-' + str(num);
- self.pwd = '/'
- self.env = {}
- self.path = []
-
- def cd(self, path):
- self.pwd = path.format(host = self.hostname)
-
- def export(self, env, value):
- self.env[env] = value.format(host = self.hostname)
-
- def add_path(self, path):
- self.path.append(path.format(host = self.hostname))
-
- def prefix(self, cmd):
- pre = []
- pre.append('cd "{0}"'.format(self.pwd))
- for (e, v) in self.env.iteritems():
- pre.append('export {0}=\\"{1}\\"'.format(e, v))
- for p in self.path:
- pre.append('export PATH="{0}:${{PATH}}"'.format(p))
- pre.append(cmd)
- return ' && '.join(pre)
-
- def run(self, cmd, warn_only = False, quiet = False, vewy_quiet = False,
- abandon_output = True):
- # Don't use single quotes in `cmd`, this will break and end badly.
- cmd = cmd.format(host = self.hostname)
- cmd = self.prefix(cmd)
- print(self.hostname + ' =>')
- if vewy_quiet:
- # Be vewy, vewy quiet, I'm hunting wabbits.
- print('[command hidden]\n')
- quiet = True
- else:
- print(cmd + '\n')
- cmd = '''ssh '{0}' "bash -c '{1}'"'''.format(self.host, cmd)
- try:
- return Process.run(cmd, quiet, abandon_output)
- except Exception as e:
- if warn_only:
- print(str(e) + '---------- This was only a warning, ' +
- 'it won\'t stop the execution --\n')
- return None
- else:
- raise e
-
-class SSHSet():
- def __init__(self, conn = []):
- self.conn = conn
-
- def __len__(self):
- return len(self.conn)
-
- def add(self, conn):
- if isinstance(conn, list):
- self.conn.extend(conn)
- else:
- self.conn.append(conn)
-
- def cd(self, path):
- for conn in self.conn:
- conn.cd(path)
-
- def export(self, env, value):
- for conn in self.conn:
- conn.export(env, value)
-
- def add_path(self, path):
- for conn in self.conn:
- conn.add_path(path)
-
- def run(self, cmd, parallel = True, quiet = False, vewy_quiet = False,
- abandon_output = True, warn_only = False):
- if not parallel:
- for conn in self.conn:
- conn.run(cmd, quiet = quiet, vewy_quiet = vewy_quiet,
- abandon_output = abandon_output, warn_only = warn_only)
- else:
- threads = []
- queue = Queue()
- def wrapper(conn, cmd, queue):
- try:
- conn.run(cmd, quiet = quiet, vewy_quiet = vewy_quiet,
- abandon_output = abandon_output,
- warn_only = warn_only)
- except Exception as e:
- queue.put(Exception(conn.hostname + ' => ' + str(e)))
- for conn in self.conn:
- thread = Thread(target = wrapper, args = (conn, cmd, queue, ))
- thread.start()
- threads.append(thread)
- for thread in threads:
- thread.join()
- if not queue.empty():
- l = []
- while not queue.empty():
- e = queue.get()
- l.append(str(e));
- raise Exception('\n'.join(l))
diff --git a/testutils/ptest/config.py b/testutils/ptest/config.py
deleted file mode 100644
index 43e5224..0000000
--- a/testutils/ptest/config.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import json
-import copy
-import getpass
-import os.path
-
-from Ssh import SSHConnection, SSHSet
-
-local = None
-qfile_set = None
-other_set = None
-remote_set = None
-all_set = None
-
-master_base_path = None
-host_base_path = None
-java_home = None
-
-def load(config_file = '~/.hive_ptest.conf'):
- global local, qfile_set, other_set, remote_set, all_set
- global master_base_path, host_base_path, java_home
-
- config_file = os.path.expanduser(config_file)
-
- cfg = None
- try:
- with open(config_file) as f:
- cfg = json.loads(f.read())
-
- host_nodes = {}
- def get_node(host):
- if not host in host_nodes:
- host_nodes[host] = -1
- host_nodes[host] += 1
- return SSHConnection(host, host_nodes[host])
-
- qfile = []
- for (host, num, ) in cfg['qfile_hosts']:
- for i in range(num):
- qfile.append(get_node(host))
-
- other = []
- for (host, num, ) in cfg['other_hosts']:
- for i in range(num):
- other.append(get_node(host))
-
- local = SSHConnection('localhost')
-
- qfile_set = SSHSet(qfile)
- other_set = SSHSet(other)
-
- # Make copies, otherwise they they will be passed by reference and
- # reused. Reuse is bad - you don't want `cd` on remote_set to affect
- # anything in the all_set.
-
- remote_set = SSHSet(copy.copy(qfile))
- remote_set.add(copy.copy(other))
-
- all_set = SSHSet(copy.copy(qfile))
- all_set.add(copy.copy(other))
- all_set.add(local)
-
- master_base_path = cfg['master_base_path']
- host_base_path = cfg['host_base_path'] + '-' + getpass.getuser()
- java_home = cfg['java_home']
- except Exception as e:
- raise Exception('Failed to parse your configuration file (' +
- config_file + '). Maybe you forgot the `--config` switch?', e)
diff --git a/testutils/ptest/hivetest.py b/testutils/ptest/hivetest.py
deleted file mode 100755
index ccc4537..0000000
--- a/testutils/ptest/hivetest.py
+++ /dev/null
@@ -1,607 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import argparse
-import time
-from threading import Thread
-import os.path
-import collections
-import re
-import os
-
-import Report
-import config
-
-# WARNING
-#
-# If you are editing this code, please be aware that commands passed to `run`
-# should not use single quotes, this will break and end badly as the final
-# command looks like `ssh 'host' 'some command - single quote will break it'`.
-# Also please be aware that `run` uses `.format` to change `{host}` in commands
-# into actual host name it is running on, running `.format` on strings using
-# `{host}`, for example including `host_code_path` will not work.
-#
-# Also this code assumes master_base_path is available to all testing machines
-# and is mounted in the same place on all of them.
-#
-# Getting rid of this restrictions without making the code much more complicated
-# is very welcome.
-
-# This is configured in user configuration file.
-
-local = None
-qfile_set = None
-other_set = None
-remote_set = None
-all_set = None
-
-master_base_path = None
-host_base_path = None
-runtest_dir = os.getcwd()
-
-# End of user configurated things.
-
-ant_path = None
-arc_path = None
-phutil_path = None
-code_path = None
-report_path = None
-host_code_path = None
-ivy_path = None
-
-def read_conf(config_file):
- global local, qfile_set, other_set, remote_set, all_set
- global master_base_path, host_base_path
- global ant_path, arc_path, phutil_path, code_path, report_path, host_code_path, ivy_path
-
- if config_file is not None:
- config.load(config_file)
- else:
- config.load()
-
- local = config.local
- qfile_set = config.qfile_set
- other_set = config.other_set
- remote_set = config.remote_set
- all_set = config.all_set
-
- master_base_path = config.master_base_path
- host_base_path = config.host_base_path
-
- if 'HIVE_PTEST_SUFFIX' in os.environ:
- suffix = os.environ['HIVE_PTEST_SUFFIX']
- master_base_path += '-' + suffix
- host_base_path += '-' + suffix
-
- ant_path = master_base_path + '/apache-ant-1.8.4'
- arc_path = master_base_path + '/arcanist'
- phutil_path = master_base_path + '/libphutil'
- code_path = master_base_path + '/trunk'
- report_path = master_base_path + '/report/' + time.strftime('%m.%d.%Y_%H:%M:%S')
- host_code_path = host_base_path + '/trunk-{host}'
- ivy_path = master_base_path + '/.ivy2'
-
- # Setup of needed environmental variables and paths
-
- # Proxy
- if args.http_proxy is not None:
- all_set.export('http_proxy', args.http_proxy + ':' + args.http_proxy_port)
- all_set.export('https_proxy', args.http_proxy + ':' + args.http_proxy_port)
- all_set.export('ANT_OPTS', get_ant_opts_proxy())
-
- # Ant
- all_set.export('ANT_HOME', ant_path)
- all_set.add_path(ant_path + '/bin')
-
- # Arcanist
- all_set.add_path(arc_path + '/bin')
-
- # Java
- all_set.export('JAVA_HOME', config.java_home)
- all_set.add_path(config.java_home + '/bin')
-
- # Hive
- remote_set.export('HIVE_HOME', host_code_path + '/build/dist')
- remote_set.add_path(host_code_path + '/build/dist/bin')
-
-def get_ant_opts_proxy():
- cmd = ' -Dhttp.proxyHost=' + args.http_proxy
- cmd += ' -Dhttp.proxyPort=' + args.http_proxy_port
- cmd += ' -Dhttps.proxyHost=' + args.http_proxy
- cmd += ' -Dhttps.proxyPort=' + args.http_proxy_port
- return cmd
-
-def get_ant():
- # Gets Ant 1.8.4 from one of Apache mirrors.
- print('\n-- Installing Ant 1.8.4\n')
-
- if local.run('test -d "{0}"'.format(ant_path), warn_only = True,
- abandon_output = False) is None:
- local.run('mkdir -p "{0}"'.format(master_base_path))
- local.cd(master_base_path)
- local.run('curl "http://apache.osuosl.org//ant/binaries/apache-ant-1.8.4-bin.tar.gz" | tar xz')
- else:
- print('\n Ant 1.8.4 already installed\n')
-
-def get_arc():
- # Gets latest Arcanist and libphtuil from their Git repositories.
- print('\n-- Updating Arcanist installation\n')
-
- if local.run('test -d "{0}"'.format(arc_path), warn_only = True,
- abandon_output = False) is None:
- local.run('mkdir -p "{0}"'.format(os.path.dirname(arc_path)))
- local.run('git clone https://github.com/facebook/arcanist.git "{0}"'
- .format(arc_path))
-
- if local.run('test -d "{0}"'.format(phutil_path), warn_only = True,
- abandon_output = False) is None:
- local.run('mkdir -p "{0}"'.format(os.path.dirname(phutil_path)))
- local.run('git clone https://github.com/facebook/libphutil.git "{0}"'
- .format(phutil_path))
-
- local.cd(arc_path)
- local.run('git pull https://github.com/facebook/arcanist.git')
- local.cd(phutil_path)
- local.run('git pull https://github.com/facebook/libphutil.git')
-
-def get_clean_hive():
- # Gets latest Hive from Apache Git repository and cleans the repository
- # (undo of any changes and removal of all generated files). Also runs
- # `arc-setup` so the repo is ready to be used.
- print('\n-- Updating Hive repo\n')
-
- local.cd(code_path)
- if local.run('test -d "{0}"'.format(code_path), warn_only = True,
- abandon_output = False) is None:
- local.run('mkdir -p "{0}"'.format(os.path.dirname(code_path)))
- local.run('git clone http://git.apache.org/hive.git "{0}"'.format(code_path))
- else:
- # Clean repo and checkout to t he last revision
- local.run('git reset --hard HEAD')
- local.run('git clean -dffx')
- local.run('git pull')
-
- local.run('ant arc-setup')
-
-def copy_local_hive():
- # Copy local repo to the destination path instead of using git clone
- if local.run('test -d "{0}"'.format(code_path), warn_only = True,
- abandon_output = False) is None:
- local.run('mkdir -p "{0}"'.format(os.path.dirname(code_path)))
- local.run('rm -rf "{0}"'.format(code_path), warn_only = True)
- local.run('mkdir -p "{0}"'.format(code_path))
- local.run('echo "{0}"'.format(runtest_dir))
- local.cd(runtest_dir)
- local.run('cp -rf * "{0}"'.format(code_path))
- local.cd(code_path)
- local.run('ant arc-setup')
-
-def prepare_for_reports():
- # Generates directories for test reports. All test nodes will copy results
- # to this directories.
- print('\n-- Creating a directory for JUnit reports\n')
- # Remove previous reports that might be there.
- local.run('rm -rf "{0}"'.format(report_path), warn_only = True)
- local.run('mkdir -p "{0}/logs"'.format(report_path))
- local.run('mkdir -p "{0}/out/clientpositive"'.format(report_path))
- local.run('mkdir -p "{0}/out/clientnegative"'.format(report_path))
-
-def patch_hive(patches = [], revision = None):
- # Applies given patches to the Hive repo. Revision means a Differential
- # revision, patches list is a list of paths to patches on local file system.
- #
- # Allowing multiple revisions and patches would complicate things a little
- # (order of applied patches should be preserved, but argparse will split
- # them into two lists) so only multiple local patches are allowed.
- # Shouldn't be a big problem as you can use `arc export` to get the patches
- # locally.
- local.cd(code_path)
- if revision is not None:
- print('\n-- Patching Hive repo using a Differential revision\n')
- revision = revision.upper()
- if not revision.startswith('D'):
- revision = 'D' + revision
- local.run('arc patch "{0}"'.format(revision))
- if patches:
- print('\n-- Patching Hive repo using a patch from local file system\n')
- for patch in patches:
- local.run('patch -rf -p0 < "{0}"'.format(patch))
-
-def build_hive():
- print('\n-- Building Hive\n')
- local.cd(code_path)
- cmd = 'ant -Divy.default.ivy.user.dir={0} '.format(ivy_path)
- if args.very_clean:
- cmd += 'very-clean '
- else:
- cmd += 'clean '
- cmd += 'package'
- local.run(cmd)
-
-def propagate_hive():
- # Expects master_base_path to be available on all test nodes in the same
- # place (for example using NFS).
- print('\n-- Propagating Hive repo to all hosts\n')
- print(host_code_path)
- print(code_path)
-
- remote_set.run('rm -rf "{0}"'.format(host_code_path))
- remote_set.run('mkdir -p "{0}"'.format(host_code_path))
- remote_set.run('cp -r "{0}/*" "{1}"'.format(
- code_path, host_code_path))
-
- # It should avoid issues with 'ivy publish' exceptions during testing phase.
- remote_set.run('cp -r "{0}" "{1}"'.format(ivy_path, host_code_path))
-
-
-def segment_tests(path):
- # Removes `.q` files that should not be run on this host. The huge shell
- # command is slow (not really suprising considering amount of forking it has
- # to do), you are welcome to make it better=).
- local.cd(code_path + path)
- tests = local.run('ls -1', quiet = True, abandon_output = False).strip().split('\n')
-
- qfile_set.cd(host_code_path + path)
- test_splits = [[] for i in range(len(qfile_set))]
- i = 0
- for test in tests:
- test_splits[i].append(test)
- i = (i + 1) % len(qfile_set)
- cmd = []
- for i in range(len(qfile_set)):
- host = qfile_set.conn[i].hostname
- cmd.append('if [[ "{host}" != "' + host + '" ]]; then rm -f ' + ' '.join(test_splits[i]) + '; fi')
- cmd = ' && '.join(cmd)
- # The command is huge and printing it out is not very useful, using wabbit
- # hunting mode.
- qfile_set.run(cmd, vewy_quiet = True)
-
-def prepare_tests():
- print('\n-- Preparing test sets on all hosts\n')
- segment_tests('/ql/src/test/queries/clientpositive')
- segment_tests('/ql/src/test/queries/clientnegative')
-
-def collect_log(name):
- # Moves JUnit log to the global logs directory.
- #
- # This has the same restriction on master_base_path as propagate_hive.
- new_name = name.split('.')
- new_name[-2] += '-{host}'
- new_name = '.'.join(new_name)
- qfile_set.cd(host_code_path + '/build/ql/test')
- # If tests failed there may be no file, so warn only if `cp` is having
- # problems.
- qfile_set.run(
- 'cp "' + name + '" "' + report_path + '/logs/' + new_name + '" || ' +
- 'touch "' + report_path + '/logs/{host}-' + name + '.fail"'
- )
- # Get the hive.log too.
- qfile_set.cd(host_code_path + '/build/ql/tmp')
- qfile_set.run('cp "hive.log" "' + report_path + '/logs/hive-{host}-' + name + '.log"',
- warn_only = True)
-
-def collect_out(name, desc_name):
- # Moves `.out` file (test output) to the global logs directory.
- #
- # This has the same restriction on master_base_path as propagate_hive.
- qfile_set.cd(host_code_path + '/' + name)
- # Warn only if no files are found.
- qfile_set.run('mkdir -p "' + report_path + '/' + desc_name + '/out/' + '"', warn_only = True)
- qfile_set.run('cp * "' + report_path + '/' + desc_name + '/out/' + '"', warn_only = True)
-
-def run_tests():
- # Runs TestCliDriver and TestNegativeCliDriver testcases.
- print('\n-- Running .q file tests on all hosts\n')
-
- # Using `quiet` because output of `ant test` is not very useful when we are
- # running on many hosts and it all gets mixed up. In case of an error
- # you'll get last lines generated by `ant test` anyway (might be totally
- # irrelevant if one of the first tests fails and Ant reports a failure after
- # running all the other test, fortunately JUnit report saves the Ant output
- # if you need it for some reason).
-
- remote_ivy_path = '$(pwd)/.ivy2'
-
- qfile_set.cd(host_code_path)
- qfile_set.run('ant -Divy.default.ivy.user.dir={0} -Dtestcase=TestCliDriver test'.format(remote_ivy_path), quiet = True, warn_only = True)
- collect_log('TEST-org.apache.hadoop.hive.cli.TestCliDriver.xml')
- collect_out('build/ql/test/logs/clientpositive', 'TestCliDriver')
-
- qfile_set.cd(host_code_path)
- qfile_set.run('ant -Divy.default.ivy.user.dir={0} -Dtestcase=TestNegativeCliDriver test'.format(remote_ivy_path), quiet = True, warn_only = True)
- collect_log('TEST-org.apache.hadoop.hive.cli.TestNegativeCliDriver.xml')
- collect_out('build/ql/test/logs/clientnegative', 'TestNegativeCliDriver')
-
-def run_other_tests():
- # Runs all other tests that run_test doesn't run.
-
- def get_other_list():
- local.cd(code_path)
- # Generate test classes in build.
- local.run('ant -Dtestcase=nothing test')
-
- if (args.singlehost):
- tests = local.run(' | '.join([
- 'find build/*/test/classes -name "Test*.class"',
- 'sed -e "s:[^/]*/::g"',
- 'grep -v TestSerDe.class',
- 'grep -v TestHiveMetaStore.class',
- 'grep -v TestBeeLineDriver.class',
- 'grep -v TestHiveServer2Concurrency.class',
- 'grep -v TestCliDriver.class',
- 'grep -v TestNegativeCliDriver.class',
- 'grep -v ".*\$.*\.class"',
- 'sed -e "s:\.class::"'
- ]), abandon_output = False)
- return tests.split()
- else:
- tests = local.run(' | '.join([
- 'find build/*/test/classes -name "Test*.class"',
- 'sed -e "s:[^/]*/::g"',
- 'grep -v TestSerDe.class',
- 'grep -v TestHiveMetaStore.class',
- 'grep -v TestBeeLineDriver.class',
- 'grep -v TestHiveServer2Concurrency.class',
- 'grep -v TestCliDriver.class',
- 'grep -v TestNegativeCliDriver.class',
- 'grep -v ".*\$.*\.class"',
- 'grep -v TestSetUGIOnBothClientServer.class',
- 'grep -v TestSetUGIOnOnlyClient.class',
- 'grep -v TestSetUGIOnOnlyServer.class',
- 'grep -v TestRemoteHiveMetaStore',
- 'grep -v TestEmbeddedHiveMetaStore',
- 'sed -e "s:\.class::"'
- ]), abandon_output = False)
- return tests.split()
-
- def segment_other():
- other_set.run('mkdir -p ' + report_path + '/TestContribCliDriver', warn_only = True)
- other_set.run('mkdir -p ' + report_path + '/TestContribCliDriver/positive', warn_only = True)
- other_set.run('mkdir -p ' + report_path + '/TestContribCliDriver/negative', warn_only = True)
- other_set.run('mkdir -p ' + report_path + '/TestHBaseCliDriver', warn_only = True)
-
- # Split all test cases between hosts.
- def get_command(test):
- return '; '.join([
- 'ant -Divy.default.ivy.user.dir=$(pwd)/.ivy2 -Dtestcase=' + test + ' test',
-
- 'cp "`find . -name "TEST-*.xml"`" "' + report_path + '/logs/" || ' +
- 'touch "' + report_path + '/logs/{host}-' + test + '.fail"',
-
- 'cp "build/ql/tmp/hive.log" "' + report_path + '/logs/hive-{host}-' + test + '.log"',
-
- 'cp "build/contrib/test/logs/contribclientnegative/*" "' + report_path + '/TestContribCliDriver/negative 2>/dev/null"',
-
- 'cp "build/contrib/test/logs/contribclientpositive/*" "' + report_path + '/TestContribCliDriver/positive 2>/dev/null"',
-
- 'cp "build/hbase-handler/test/logs/hbase-handler/*" "' + report_path + '/TestHBaseCliDriver/ 2>/dev/null"'
- ])
-
- cmd = []
- i = 0
- for test in get_other_list():
- # Special case, don't run minimr tests in parallel. They will run
- # on the first host, and no other tests will run there (unless we
- # have a single host).
- #
- # TODO: Real fix would be to allow parallel runs of minimr tests.
- if len(other_set) > 1:
- if re.match('.*minimr.*', test.lower()):
- host = other_set.conn[0].hostname
- else:
- i = (i + 1) % len(other_set)
- if i == 0:
- i = 1
- host = other_set.conn[i].hostname
- else:
- # We are running on single host.
- host = other_set.conn[0].hostname
- cmd.append(
- 'if [[ "{host}" == "' + host + '" ]]; then ' +
- get_command(test) +
- '; fi'
- )
- return ' ; '.join(cmd)
-
- command = segment_other()
- other_set.cd(host_code_path)
- # See comment about quiet option in run_tests.
- other_set.run(command, quiet = True, warn_only = True)
-
-def generate_report(one_file_report = False):
- # Uses `Report.py` to create a HTML report.
- print('\n-- Generating a test report\n')
- local.run('cp "' + master_base_path + '/templogs/* " "'+ report_path + '/logs/" ', warn_only = True)
-
- # Call format to remove '{{' and '}}'.
- path = os.path.expandvars(report_path.format())
- CmdArgs = collections.namedtuple('CmdArgs', ['one_file', 'log_dir', 'report_dir'])
- args = CmdArgs(
- one_file = one_file_report,
- log_dir = '{0}/logs'.format(path),
- report_dir = path
- )
- Report.make_report(args)
-
- print('\n-- Test report has been generated and is available here:')
- print('-- "{0}/report.html"'.format(path))
- print()
-
-def stop_tests():
- # Brutally stops tests on all hosts, something more subtle would be nice and
- # would allow the same user to run this script multiple times
- # simultaneously.
- print('\n-- Stopping tests on all hosts\n')
- remote_set.run('killall -9 java', warn_only = True)
-
-def remove_code():
- # Running this only on one connection per host so there are no conflicts
- # between several `rm` calls. This removes all repositories, it would have
- # to be changed if we were to allow multiple simultaneous runs of this
- # script.
-
- print('\n-- Removing Hive code from all hosts\n')
- # We could remove only `host_code_path`, but then we would have abandoned
- # directories after lowering number of processes running on one host.
- cmd = 'rm -rf "' + host_base_path + '"'
- cmd = 'if [[ `echo "{host}" | grep -q -- "-0$"; echo "$?"` -eq "0" ]]; then ' + \
- cmd + '; fi'
- remote_set.run(cmd)
-
-def overwrite_results():
- # Copy generated `.q.out` files to master repo.
-
- local.cd(code_path)
- expanded_path = local.run('pwd', abandon_output = False)
- print('\n-- Copying generated `.q.out` files to master repository: ' +
- expanded_path)
-
- for name in ['clientpositive', 'clientnegative']:
- local.cd(report_path + '/out/' + name)
- # Don't panic if no files are found.
- local.run('cp * "' + code_path + '/ql/src/test/results/' + name + '"',
- warn_only = True)
-
-def save_svn_info():
- if args.svn_info:
- local.cd(master_base_path + '/trunk')
- local.run('git show --summary > "{0}"'.format(report_path + '/svn-info'))
-
-def save_patch():
- if args.save_patch:
- local.cd(code_path)
- local.run('git add --all')
- local.run('git diff --no-prefix HEAD > "{0}"'.format(report_path + '/patch'))
-
-# -- Tasks that can be called from command line start here.
-
-def cmd_prepare(patches = [], revision = None):
- get_ant()
- get_arc()
- if (args.copylocal):
- copy_local_hive()
- else :
- get_clean_hive()
- patch_hive(patches, revision)
-
- build_hive()
- propagate_hive()
- prepare_tests()
-
-def cmd_run_tests(one_file_report = False):
- prepare_for_reports()
- save_svn_info()
- save_patch()
-
- t = Thread(target = run_other_tests)
- t.start()
- run_tests()
- t.join()
-
- if args.overwrite:
- overwrite_results()
-
- generate_report(one_file_report)
-
-def cmd_test(patches = [], revision = None, one_file_report = False):
- cmd_prepare(patches, revision)
-
- if args.singlehost==False:
- local.cd(master_base_path + '/trunk')
- local.run('chmod -R 777 *');
- local.run('rm -rf "' + master_base_path + '/templogs/"')
- local.run('mkdir -p "' + master_base_path + '/templogs/"')
- tests = ['TestRemoteHiveMetaStore','TestEmbeddedHiveMetaStore','TestSetUGIOnBothClientServer','TestSetUGIOnOnlyClient','TestSetUGIOnOnlyServer']
-
- for test in tests:
- local.run('sudo -u root ant -Divy.default.ivy.user.dir={0} '.format(ivy_path) + ' -Dtestcase=' + test + ' test')
- local.run('cp "`find . -name "TEST-*.xml"`" "' + master_base_path + '/templogs/"')
-
- cmd_run_tests(one_file_report)
-
-def cmd_stop():
- stop_tests()
-
-def cmd_remove():
- remove_code()
-
-parser = argparse.ArgumentParser(description =
- 'Hive test farm controller.')
-parser.add_argument('--config', dest = 'config',
- help = 'Path to configuration file')
-parser.add_argument('--prepare', action = 'store_true', dest = 'prepare',
- help = 'Builds Hive and propagates it to all test machines')
-parser.add_argument('--run-tests', action = 'store_true', dest = 'run_tests',
- help = 'Runs tests on all test machines')
-parser.add_argument('--test', action = 'store_true', dest = 'test',
- help = 'Same as running `prepare` and then `run-tests`')
-parser.add_argument('--report-name', dest = 'report_name',
- help = 'Store report and logs directory called `REPORT_NAME`')
-parser.add_argument('--stop', action = 'store_true', dest = 'stop',
- help = 'Kill misbehaving tests on all machines')
-parser.add_argument('--remove', action = 'store_true', dest = 'remove',
- help = 'Remove Hive trunk copies from test machines')
-parser.add_argument('--revision', dest = 'revision',
- help = 'Differential revision to test')
-parser.add_argument('--patch', dest = 'patch', nargs = '*',
- help = 'Patches from local file system to test')
-parser.add_argument('--one-file-report', dest = 'one_file_report',
- action = 'store_true',
- help = 'Generate one (huge) report file instead of multiple small ones')
-parser.add_argument('--overwrite', dest = 'overwrite', action = 'store_true',
- help = 'Overwrite result files in master repo')
-parser.add_argument('--copylocal', dest = 'copylocal', action = 'store_true',
- help = 'Copy local repo instead of using git clone and git hub')
-parser.add_argument('--singlehost', dest = 'singlehost', action = 'store_true',
- help = 'Only run the test on single host, It is the users '
- 'responsibility to make sure that the conf. file does not '
- 'contain multiple hosts. '
- 'The script is not doing any validation. When --singlehost is set '
- 'the script should not be run using sudo.')
-parser.add_argument('--very-clean', action = 'store_true', dest = 'very_clean',
- help = 'Build hive with `very-clean` option')
-parser.add_argument('--svn-info', dest = 'svn_info', action = 'store_true',
- help = 'Save result of `svn info` into ${report_path}/svn-info')
-parser.add_argument('--save-patch', dest = 'save_patch', action = 'store_true',
- help = 'Save applied patch into ${report_path}/patch')
-parser.add_argument('--http-proxy', dest = 'http_proxy',
- help = 'Proxy host')
-parser.add_argument('--http-proxy-port', dest = 'http_proxy_port',
- help = 'Proxy port')
-
-args = parser.parse_args()
-
-read_conf(args.config)
-
-if args.report_name:
- report_path = '/'.join(report_path.split('/')[:-1] + [args.report_name])
-
-if args.prepare:
- cmd_prepare(args.patch, args.revision)
-elif args.run_tests:
- cmd_run_tests(args.one_file_report)
-elif args.test:
- cmd_test(args.patch, args.revision, args.one_file_report)
-elif args.stop:
- cmd_stop()
-elif args.remove:
- cmd_remove()
-else:
- parser.print_help()
-
diff --git a/testutils/ptest/templates/Properties.html b/testutils/ptest/templates/Properties.html
deleted file mode 100644
index c299129..0000000
--- a/testutils/ptest/templates/Properties.html
+++ /dev/null
@@ -1,41 +0,0 @@
-<!doctype html>
-<!--
- - Licensed to the Apache Software Foundation (ASF) under one or more
- - contributor license agreements. See the NOTICE file distributed with
- - this work for additional information regarding copyright ownership.
- - The ASF licenses this file to You under the Apache License, Version 2.0
- - (the "License"); you may not use this file except in compliance with
- - the License. You may obtain a copy of the License at
- -
- - http://www.apache.org/licenses/LICENSE-2.0
- -
- - Unless required by applicable law or agreed to in writing, software
- - distributed under the License is distributed on an "AS IS" BASIS,
- - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- - See the License for the specific language governing permissions and
- - limitations under the License.
--->
-<html>
- <head>
- <title>${this.host_name()} - Properties</title>
- <style type="text/css">
- ${this.render('templates/common.css')}
- </style>
- </head>
- <body>
- % if this.properties:
- <table class="properties">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- % for (name, value, ) in this.properties.iteritems():
- <tr>
- <td>${name}</td>
- <td class="long-lines">${value}</td>
- </tr>
- % endfor
- </table>
- % endif
- </body>
-</html>
diff --git a/testutils/ptest/templates/TestCase.html b/testutils/ptest/templates/TestCase.html
deleted file mode 100644
index 255caf8..0000000
--- a/testutils/ptest/templates/TestCase.html
+++ /dev/null
@@ -1,43 +0,0 @@
-<!--
- - Licensed to the Apache Software Foundation (ASF) under one or more
- - contributor license agreements. See the NOTICE file distributed with
- - this work for additional information regarding copyright ownership.
- - The ASF licenses this file to You under the Apache License, Version 2.0
- - (the "License"); you may not use this file except in compliance with
- - the License. You may obtain a copy of the License at
- -
- - http://www.apache.org/licenses/LICENSE-2.0
- -
- - Unless required by applicable law or agreed to in writing, software
- - distributed under the License is distributed on an "AS IS" BASIS,
- - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- - See the License for the specific language governing permissions and
- - limitations under the License.
--->
-
-<tr
- % if this.failure() or this.error():
- class="failure"
- % endif
->
- <td>${this.get_name()}</td>
- <td>
- % if this.failure():
- Failed
- % elif this.error():
- Error
- % else:
- Success
- % endif
- </td>
- <td>${this.get_time()}s</td>
-</tr>
-% if this.failure() or this.error():
- <tr class="failure">
- <td colspan="3">
- <pre>
-${this.get_log()}
- </pre>
- </td>
- </tr>
-% endif
diff --git a/testutils/ptest/templates/TestRun.html b/testutils/ptest/templates/TestRun.html
deleted file mode 100644
index 756437d..0000000
--- a/testutils/ptest/templates/TestRun.html
+++ /dev/null
@@ -1,90 +0,0 @@
-<!doctype html>
-<!--
- - Licensed to the Apache Software Foundation (ASF) under one or more
- - contributor license agreements. See the NOTICE file distributed with
- - this work for additional information regarding copyright ownership.
- - The ASF licenses this file to You under the Apache License, Version 2.0
- - (the "License"); you may not use this file except in compliance with
- - the License. You may obtain a copy of the License at
- -
- - http://www.apache.org/licenses/LICENSE-2.0
- -
- - Unless required by applicable law or agreed to in writing, software
- - distributed under the License is distributed on an "AS IS" BASIS,
- - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- - See the License for the specific language governing permissions and
- - limitations under the License.
--->
-<html>
- <head>
- <title>Unit Test Results.</title>
- <style type="text/css">
- ${this.render('templates/common.css')}
- </style>
- </head>
- <body>
- <h1 id="top">Unit Test Results.</h1>
-
- <h2>Summary</h2>
- <table>
- <tr>
- <th>Tests</th>
- <th>Passes</th>
- <th>Failures</th>
- <th>Errors</th>
- <th>Success rate</th>
- <th>Time</th>
- </tr>
- <tr
- % if this.failures() + this.errors():
- class="failure"
- % endif
- >
- <td>${this.tests()}</td>
- <td>${this.passes()}</td>
- <td>${this.failures()}</td>
- <td>${this.errors()}</td>
- <td>${round(this.success_rate(), 2)}%</td>
- <td>${round(this.time(), 2)}s</td>
- </tr>
- </table>
-
- % if this.test_suites:
- <h2>Test results</h2>
- <table>
- <tr>
- <th>Host</th>
- <th>Name</th>
- <th>Tests</th>
- <th>Passes</th>
- <th>Failures</th>
- <th>Errors</th>
- <th>Time</th>
- </tr>
- % for test_suite in this.test_suites:
- <tr
- % if test_suite.failures() + test_suite.errors():
- class="failure"
- % endif
- >
- <td>
- <a href="#${test_suite.label()}">
- ${test_suite.host_name()}
- </a>
- </td>
- <td>${test_suite.name()}</td>
- <td>${test_suite.tests()}</td>
- <td>${test_suite.passes()}</td>
- <td>${test_suite.failures()}</td>
- <td>${test_suite.errors()}</td>
- <td>${round(test_suite.time(), 2)}s</td>
- </tr>
- % endfor
- </table>
- % endif
-
- % for test_suite in this.test_suites:
- ${test_suite.render()}
- % endfor
- </body>
-</html>
diff --git a/testutils/ptest/templates/TestSuite.html b/testutils/ptest/templates/TestSuite.html
deleted file mode 100644
index 563054c..0000000
--- a/testutils/ptest/templates/TestSuite.html
+++ /dev/null
@@ -1,37 +0,0 @@
-<!--
- - Licensed to the Apache Software Foundation (ASF) under one or more
- - contributor license agreements. See the NOTICE file distributed with
- - this work for additional information regarding copyright ownership.
- - The ASF licenses this file to You under the Apache License, Version 2.0
- - (the "License"); you may not use this file except in compliance with
- - the License. You may obtain a copy of the License at
- -
- - http://www.apache.org/licenses/LICENSE-2.0
- -
- - Unless required by applicable law or agreed to in writing, software
- - distributed under the License is distributed on an "AS IS" BASIS,
- - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- - See the License for the specific language governing permissions and
- - limitations under the License.
--->
-
-<h3 id="${this.label()}">${this.label()}</h3>
-% if this.test_cases:
- <table>
- <tr>
- <th class="wide">Name</th>
- <th>Status</th>
- <th>Time</th>
- </tr>
- % for test_case in this.test_cases:
- ${test_case.render()}
- % endfor
- </table>
-% endif
-<a
- class="float-right"
- href="${this.render_link(this.label() + '-properties.html', 'templates/Properties.html')}"
->
- Properties >
-</a>
-<a href="#top">Back to top</a>
diff --git a/testutils/ptest/templates/common.css b/testutils/ptest/templates/common.css
deleted file mode 100644
index 256967f..0000000
--- a/testutils/ptest/templates/common.css
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-body {
- font-size: 0.85em;
-}
-
-a {
- color: blue;
-}
-
-table {
- width: 100%;
-}
-
-table.properties {
- table-layout: fixed;
-}
-
-table.properties th:first-child {
- width: 30%;
-}
-
-table.properties th:first-child + th {
- width: 70%;
-}
-
-table th {
- background-color: #a6caf0;
- padding: 0.3em;
- text-align: left;
-}
-
-table td {
- background-color: #eeeee0;
- padding: 0.3em;
-}
-
-table tr.failure {
- color: red;
- font-weight: bold;
-}
-
-table tr.failure a {
- color: red;
-}
-
-.long-lines {
- word-wrap: break-word;
-}
-
-.float-right {
- float: right;
-}
-
-.wide {
- width: 100%;
-}
diff --git a/testutils/ptest2/.gitignore b/testutils/ptest2/.gitignore
deleted file mode 100644
index 4a66dab..0000000
--- a/testutils/ptest2/.gitignore
+++ /dev/null
@@ -1,11 +0,0 @@
-/.project
-/.settings
-/.classpath
-/build
-/.idea/
-/logs
-/target
-*.iml
-*.orig
-*~
-velocity.log*
diff --git a/testutils/ptest2/README.md b/testutils/ptest2/README.md
deleted file mode 100644
index f7d6d18..0000000
--- a/testutils/ptest2/README.md
+++ /dev/null
@@ -1,100 +0,0 @@
-# About
-
-Hive-ptest is a parallel testing framework for executing the Hive unit tests. If you'd like
-to dive into the code, start with org.apache.hive.ptest.RunTests.
-
-# Building
-
-Approvals is required for testing and must be manually installed:
-
-
-http://sourceforge.net/projects/approvaltests/files/ApprovalTests.Java/
-
-Example:
-
-cd /tmp
-wget http://people.apache.org/~brock/ApprovalTests.013/ApprovalTests.jar
-mvn install:install-file -Dfile=/tmp/ApprovalTests.jar -DgroupId=approvaltests \
- -DartifactId=ApprovalsJava -Dversion=013 -Dpackaging=jar
-
-# Building
-
- mvn clean package
-
-# Sizing
-
-We utilize 8 servers for this process and trunk builds complete in 1 hour. Each slave
-has 8 physical cores with hyperthreading enabled and 48GB of ram. Each slave is allowed
-8 test "threads". I had used more than 8 threads but Zookeeper timed out randomly.
-
-# Configuring
-
-* Create a user such as hiveptest on the master and all slaves.
-* Setup passwordless ssh form the master to all slaves.
-* Ensure that SSH connection attempts won't fail.
-
-On all slaves add the following to /etc/ssh/sshd_config:
-
- MaxAuthTries 100
- MaxSessions 100
- MaxStartups 100
-
-# Install git, svn, make, patch, java, ant, and maven
-
-Recent version of git, svn, make, patch, java, ant and maven should be installed. Additionally
-environment variables such as MAVEN_OPTS and ANT_OPTS should be configured with large leap sizes:
-
- $ for item in java maven ant; do echo $item; cat /etc/profile.d/${item}.sh;done
- java
- export JAVA_HOME=$(readlink -f /usr/java/default)
- export PATH=$JAVA_HOME/bin:$PATH
- maven
- export M2_HOME=$(readlink -f /usr/local/apache-maven)
- export PATH=$M2_HOME/bin:$PATH
- export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=256M"
- ant
- export ANT_HOME=$(readlink -f /usr/local/apache-ant)
- export PATH=$ANT_HOME/bin:$PATH
- export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m"
-
-# Ensure umask is setup to 0022
-
- $ cat /etc/profile.d/umask.sh
- umask 0022
-
-# On all slaves ensure that nproc and nofile are set to high values
-
-# rm /etc/security/limits.d/90-nproc.conf
-
-# cat /etc/security/limits.d/nproc.conf
-
-* soft nproc 32768
-* hard nproc 65536
-
-root soft nproc 32768
-root hard nproc 65536
-# cat /etc/security/limits.d/nofile.conf
-
-* soft nofile 32768
-* hard nofile 32768
-
-root soft nofile 32768
-root hard nofile 32768
-
-# If using the Cloud Host Provider:
-
-Ensure the user running the tests has strict host/key checking disabled:
-
- $ cat ~/.ssh/config
- StrictHostKeyChecking no
- ConnectTimeout 20
- ServerAliveInterval 1
-
-# Configure properties file
-
-See conf/example-apache-trunk.properties
-
-# Execute
-
- mvn dependency:copy-dependencies
- java -Xms4g -Xmx4g -cp "target/hive-ptest-1.0-classes.jar:target/dependency/*" org.apache.hive.ptest.execution.PTest --properties apache-trunk.properties
diff --git a/testutils/ptest2/conf/cloudhost.properties.example b/testutils/ptest2/conf/cloudhost.properties.example
deleted file mode 100644
index c375eb5..0000000
--- a/testutils/ptest2/conf/cloudhost.properties.example
+++ /dev/null
@@ -1,54 +0,0 @@
-#/**
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-#
-# This is just an example of different cloudhost providers
-#
-
-# This context provides configurations for AWS EC2 and GCE (google compute engine)
-executionContextProvider = org.apache.hive.ptest.execution.context.CloudExecutionContextProvider$Builder
-
-# Option: GCE
-cloudProvider = google-compute-engine
-gceJsonFile = # GCE JSON KEY FILE
-instanceType = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/zones/us-central1-a/machineTypes/n1-standard-8
-imageId = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/global/images/hive-ptest-debian-8-20160927
-# keyPair = # UNUSED
-securityGroup = hive-ptest
-
-# Option: AWS
-cloudProvider = aws-ec2
-apiKey = # AWS ACCESS KEY
-accessKey = # AWS SECRET ACCESS KEY
-instanceType = c3.2xlarge
-imageId = us-west-1/ami-1fa1445b
-keyPair = hive-ptest
-securityGroup = hive-ptest
-
-# Generic options
-workingDirectory = /data/hive-ptest
-profileDirectory = /usr/local/hiveptest/etc/public/
-privateKey = /home/hiveptest/.ssh/hive-ptest-user-key
-dataDir = /data/hive-ptest/data/
-numHosts = 12
-groupName = hive-ptest-slaves
-localDirs = /home/hiveptest/
-user = hiveptest
-numThreads = 2
-maxLogDirectoriesPerProfile = 30
-userMetadata.owner = # USER
-maxHostsPerCreateRequest = 12
diff --git a/testutils/ptest2/conf/deployed/master-mr2.properties b/testutils/ptest2/conf/deployed/master-mr2.properties
deleted file mode 100644
index d269c7b..0000000
--- a/testutils/ptest2/conf/deployed/master-mr2.properties
+++ /dev/null
@@ -1,222 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-repositoryType = git
-repository = https://github.com/apache/hive.git
-repositoryName = apache-github-source
-branch = master
-
-jiraUrl = https://issues.apache.org/jira
-jiraUser = hiveqa
-#jiraPassword = #TODO Set this value
-jenkinsURL = https://builds.apache.org/job
-logsURL = http://104.198.109.242/logs/
-
-mavenArgs =
-#mavenEnvOpts = -Dhttp.proxyHost=localhost -Dhttp.proxyPort=3128
-mavenTestArgs =
-testCasePropertyName = test
-buildTool = maven
-javaHome = /usr/lib/jvm/java-8-openjdk-amd64
-javaHomeForTests = /usr/lib/jvm/java-8-openjdk-amd64
-#javaHomeForTests = /usr/java/jdk1.8.0_25
-unitTests.directories = ./
-additionalProfiles =
-
-unitTests.isolate = TestAuthorizationPreEventListener TestDefaultHiveMetastoreAuthorizationProvider TestEmbeddedHiveMetaStore TestExecDriver TestHive TestHiveMetaStoreWithEnvironmentContext TestHiveRemote TestMetaStoreAuthorization TestMetaStoreEventListener TestRemoteHiveMetaStore TestRemoteHiveMetaStoreIpAddress TestRemoteUGIHiveMetaStoreIpAddress TestSetUGIOnBothClientServer TestSetUGIOnOnlyClient TestSetUGIOnOnlyServer TestZNodeSetUp TestHCatStorer TestHCatLoader TestHCatExternalDyna [...]
-
-# comes from build-command.xml excludes
-unitTests.exclude = TestSerDe TestHiveMetaStore TestHiveServer2Concurrency
-
-unitTests.batchSize=20
-
-unitTests.module.accumulo-handler=accumulo-handler
-ut.accumulo-handler.batchSize=0
-
-unitTests.module.common=common
-ut.common.batchSize=0
-
-unitTests.module.hbase-handler=hbase-handler
-ut.hbase-handler.batchSize=0
-
-unitTests.module.metastore=metastore
-ut.metastore.batchSize=12
-
-
-unitTests.module.ql=ql
-ut.ql.batchSize=10
-ut.ql.skipBatching=TestDbTxnManager2 TestTxnCommands2WithSplitUpdateAndVectorization TestTxnCommands2WithSplitUpdate
-
-unitTests.module.service=service
-ut.service.batchSize=8
-
-unitTests.module.itests.hive-unit=itests.hive-unit
-ut.itests.hive-unit.batchSize=9
-ut.itests.hive-unit.skipBatching=TestAcidOnTezWithSplitUpdate TestAcidOnTez TestMTQueries \
- TestCompactor TestSchedulerQueue TestOperationLoggingAPIWithTez TestSSL TestJdbcDriver2 \
- TestJdbcWithMiniHA TestJdbcWithMiniMr TestReplicationScenariosIncrementalLoadAcidTables \
- TestReplIncrementalLoadAcidTablesWithJsonMessage TestReplicationScenarios \
- TestReplWithJsonMessageFormat TestReplWithJsonMessageFormat \
- TestStatsReplicationScenariosACIDNoAutogather TestStatsReplicationScenariosMMNoAutogather \
- TestStatsReplicationScenariosACID TestStatsReplicationScenariosMM \
- TestStatsReplicationScenariosNoAutogather TestStatsReplicationScenarios \
- TestReplicationScenariosAcidTables TestReplAcidTablesBootstrapWithJsonMessage \
- TestReplicationScenariosAcidTablesBootstrap TestReplAcidTablesWithJsonMessage \
- TestTableLevelReplicationScenarios \
- TestReplicationScenariosExternalTables TestReplAcrossInstancesWithJsonMessageFormat \
- TestReplicationOnHDFSEncryptedZonesTestReplAcidTablesWithJsonMessage \
- TestReplicationScenariosAcrossInstances TestReplicationOfHiveStreaming TestReplScenariosWithStrictManaged TestReplChangeManager
-
-unitTests.module.itests.qtest=itests.qtest
-ut.itests.qtest.batchSize=9
-ut.itests.qtest.skipBatching=TestContribCliDriver TestContribNegativeCliDriver TestEncryptedHDFSCliDriver TestHBaseNegativeCliDriver TestPerfCliDriver TestParseNegativeDriver
-
-unitTests.module.hcatalog.core=hcatalog.core
-ut.hcatalog.core.batchSize=9
-
-
-qFileTests = clientPositive miniMr clientNegative miniMrNegative hbasePositive miniTez spark miniLlap miniLlapLocal miniLlapExtDB encryptedCli miniSparkOnYarn
-qFileTests.propertyFiles.mainProperties = itests/src/test/resources/testconfiguration.properties
-
-qFileTest.clientPositive.driver = TestCliDriver
-qFileTest.clientPositive.directory = ql/src/test/queries/clientpositive
-qFileTest.clientPositive.batchSize = 30
-qFileTest.clientPositive.queryFilesProperty = qfile
-qFileTest.clientPositive.exclude = minimr
-qFileTest.clientPositive.groups.minimr = mainProperties.${minimr.query.files}
-
-qFileTest.miniMr.driver = TestMinimrCliDriver
-qFileTest.miniMr.directory = ql/src/test/queries/clientpositive
-qFileTest.miniMr.batchSize = 10
-qFileTest.miniMr.queryFilesProperty = minimr.query.files
-qFileTest.miniMr.include = normal
-qFileTest.miniMr.isolate = flaky
-# normal are tests that run in minimr mode via build-common.xml
-qFileTest.miniMr.groups.normal = mainProperties.${minimr.query.files}
-
-qFileTest.encryptedCli.driver = TestEncryptedHDFSCliDriver
-qFileTest.encryptedCli.directory = ql/src/test/queries/clientpositive
-qFileTest.encryptedCli.batchSize = 6
-qFileTest.encryptedCli.queryFilesProperty = encrypted.query.files
-qFileTest.encryptedCli.include = normal
-qFileTest.encryptedCli.isolate = long
-qFileTest.encryptedCli.groups.normal = mainProperties.${encrypted.query.files}
-qFileTest.encryptedCli.groups.long = encryption_insert_partition_dynamic.q encryption_insert_partition_static.q
-
-qFileTest.miniMrNegative.driver = TestNegativeMinimrCliDriver
-qFileTest.miniMrNegative.directory = ql/src/test/queries/clientnegative
-qFileTest.miniMrNegative.batchSize = 1000
-qFileTest.miniMrNegative.queryFilesProperty = minimr.query.negative.files
-qFileTest.miniMrNegative.include = normal
-qFileTest.miniMrNegative.isolate = flaky
-# normal are tests that run in minimr mode via build-common.xml
-qFileTest.miniMrNegative.groups.normal = mainProperties.${minimr.query.negative.files}
-
-qFileTest.clientNegative.driver = TestNegativeCliDriver
-qFileTest.clientNegative.directory = ql/src/test/queries/clientnegative
-qFileTest.clientNegative.batchSize = 400
-qFileTest.clientNegative.queryFilesProperty = qfile
-qFileTest.clientNegative.exclude = miniMrNormal failing
-# stats_aggregator_error_1.q fails in both negative client and minimr mode
-# Disable for HIVE-4941 as this tests runs via ant test
-#qFileTest.clientNegative.groups.failing = stats_aggregator_error_1.q
-# normal are run via minimr
-qFileTest.clientNegative.groups.miniMrNormal = cluster_tasklog_retrieval.q minimr_broken_pipe.q mapreduce_stack_trace.q mapreduce_stack_trace_turnoff.q mapreduce_stack_trace_turnoff_hadoop20.q
-
-qFileTest.hbasePositive.driver = TestHBaseCliDriver
-qFileTest.hbasePositive.directory = hbase-handler/src/test/queries/positive
-qFileTest.hbasePositive.batchSize = 5
-qFileTest.hbasePositive.queryFilesProperty = qfile
-qFileTest.hbasePositive.isolate = long
-qFileTest.hbasePositive.exclude = minimr
-#qFileTest.hbasePositive.groups.long = hbase_queries.q hbase_binary_storage_queries.q hbase_binary_map_queries.q hbase_joins.q hbase_bulk.q
-
-qFileTest.miniTez.driver = TestMiniTezCliDriver
-qFileTest.miniTez.directory = ql/src/test/queries/clientpositive
-qFileTest.miniTez.batchSize = 15
-qFileTest.miniTez.queryFilesProperty = qfile
-qFileTest.miniTez.include = normal
-#qFileTest.miniTez.exclude = HIVE-8964
-qFileTest.miniTez.groups.normal = mainProperties.${minitez.query.files} mainProperties.${minitez.query.files.shared}
-#qFileTest.miniTez.groups.HIVE-8964 = lvj_mapjoin.q
-
-qFileTest.spark.driver = TestSparkCliDriver
-qFileTest.spark.directory = ql/src/test/queries/clientpositive
-qFileTest.spark.batchSize = 15
-qFileTest.spark.queryFilesProperty = qfile
-qFileTest.spark.include = normal
-qFileTest.spark.groups.normal = mainProperties.${spark.query.files} mainProperties.${spark.only.query.files}
-
-qFileTest.miniSparkOnYarn.driver = TestMiniSparkOnYarnCliDriver
-qFileTest.miniSparkOnYarn.directory = ql/src/test/queries/clientpositive
-qFileTest.miniSparkOnYarn.batchSize = 20
-qFileTest.miniSparkOnYarn.queryFilesProperty = qfile
-qFileTest.miniSparkOnYarn.include = normal
-qFileTest.miniSparkOnYarn.groups.normal = mainProperties.${miniSparkOnYarn.query.files} mainProperties.${miniSparkOnYarn.only.query.files}
-
-qFileTest.miniLlap.driver = TestMiniLlapCliDriver
-qFileTest.miniLlap.directory = ql/src/test/queries/clientpositive
-qFileTest.miniLlap.batchSize = 15
-qFileTest.miniLlap.queryFilesProperty = qfile
-qFileTest.miniLlap.include = normal
-qFileTest.miniLlap.isolate = flaky
-qFileTest.miniLlap.groups.normal = mainProperties.${minillap.query.files} mainProperties.${minillap.shared.query.files}
-qFileTest.miniLlap.groups.flaky = acid_bucket_pruning.q
-
-qFileTest.miniLlapLocal.driver = TestMiniLlapLocalCliDriver
-qFileTest.miniLlapLocal.directory = ql/src/test/queries/clientpositive
-qFileTest.miniLlapLocal.batchSize = 30
-qFileTest.miniLlapLocal.queryFilesProperty = qfile
-qFileTest.miniLlapLocal.include = normal
-qFileTest.miniLlapLocal.groups.normal = mainProperties.${minillaplocal.query.files} mainProperties.${minillaplocal.shared.query.files}
-
-qFileTest.miniLlapExtDB.driver = TestMiniLlapExtDBCliDriver
-qFileTest.miniLlapExtDB.directory = ql/src/test/queries/clientpositive
-qFileTest.miniLlapExtDB.batchSize = 30
-qFileTest.miniLlapExtDB.queryFilesProperty = qfile
-qFileTest.miniLlapExtDB.include = normal
-qFileTest.miniLlapExtDB.groups.normal = mainProperties.${externalDB.llap.query.files}
-
-qFileTest.erasurecodingCli.driver = TestErasureCodingHDFSCliDriver
-qFileTest.erasurecodingCli.directory = ql/src/test/queries/clientpositive
-qFileTest.erasurecodingCli.batchSize = 15
-qFileTest.erasurecodingCli.queryFilesProperty = qfile
-qFileTest.erasurecodingCli.include = normal
-qFileTest.erasurecodingCli.groups.normal = mainProperties.${erasurecoding.only.query.files} mainProperties.${erasurecoding.shared.query.files}
-
-qFileTest.miniDruid.driver = TestMiniDruidCliDriver
-qFileTest.miniDruid.directory = ql/src/test/queries/clientpositive
-qFileTest.miniDruid.batchSize = 55
-qFileTest.miniDruid.queryFilesProperty = qfile
-qFileTest.miniDruid.include = normal
-qFileTest.miniDruid.groups.normal = mainProperties.${druid.query.files}
-
-qFileTest.miniDruidKafka.driver = TestMiniDruidKafkaCliDriver
-qFileTest.miniDruidKafka.directory = ql/src/test/queries/clientpositive
-qFileTest.miniDruidKafka.batchSize = 15
-qFileTest.miniDruidKafka.queryFilesProperty = qfile
-qFileTest.miniDruidKafka.include = normal
-qFileTest.miniDruidKafka.groups.normal = mainProperties.${druid.kafka.query.files}
-qFileTest.miniDruidKafka.groups.flaky = druidkafkamini_avro.q druidkafkamini_csv.q druid_timestamptz2.q druidmini_extractTime.q druidmini_floorTime.q druidmini_mv.q druidmini_semijoin_reduction_all_types.q druidmini_test1.q druidmini_test_alter.q druidmini_test_insert.q druidmini_test_ts.q druidmini_expressions.q druidmini_joins.q druidmini_masking.q
-
-qFileTest.miniKafka.driver = TestMiniHiveKafkaCliDriver
-qFileTest.miniKafka.directory = ql/src/test/queries/clientpositive
-qFileTest.miniKafka.batchSize = 15
-qFileTest.miniKafka.queryFilesProperty = qfile
-qFileTest.miniKafka.include = normal
-qFileTest.miniKafka.groups.normal = mainProperties.${hive.kafka.query.files}
diff --git a/testutils/ptest2/conf/example-apache-trunk.properties b/testutils/ptest2/conf/example-apache-trunk.properties
deleted file mode 100644
index bb6952c..0000000
--- a/testutils/ptest2/conf/example-apache-trunk.properties
+++ /dev/null
@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This configuration has two hosts
-hosts = hiveptest-2 hiveptest-3
-host.hiveptest-2.user = hiveptest
-host.hiveptest-2.host = hiveptest-2.some.domain.com
-host.hiveptest-2.threads = 2
-# Each host has two disks
-host.hiveptest-2.localDirs = /media/ephemeral0/hiveptest/,/media/ephemeral1/hiveptest/
-host.hiveptest-3.user = hiveptest
-host.hiveptest-3.host = hiveptest-3.some.domain.com
-host.hiveptest-3.threads = 2
-host.hiveptest-3.localDirs = /media/ephemeral0/hiveptest/,/media/ephemeral1/hiveptest/
-
-privateKey = /home/hiveptest/.ssh/id_rsa
-repository = git://github.com/apache/hive.git
-repositoryName = apache-github
-branch = trunk
-workingDirectory = /media/ephemeral0/hiveptest/
-antArgs = -Dtest.continue.on.failure=true
-# antEnvOpts can be used to set ANT_OPTS properties
-
-unitTests.directories = build/anttasks/test/classes build/beeline/test/classes build/cli/test/classes \
- build/common/test/classes build/contrib/test/classes build/hbase-handler/test/classes \
- build/hwi/test/classes build/jdbc/test/classes build/metastore/test/classes \
- build/ql/test/classes build/serde/test/classes build/service/test/classes build/shims/test/classes \
- build/testutils/test/classes \
- hcatalog/core/build/test/classes hcatalog/hcatalog-pig-adapter/build/test/classes \
- hcatalog/server-extensions/build/test/classes hcatalog/webhcat/svr/build/test/classes \
- hcatalog/webhcat/java-client/build/test/classes hcatalog/storage-handlers/hbase/build/test/classes
-
-unitTests.isolate = TestRemoteHiveMetaStore TestEmbeddedHiveMetaStore TestSetUGIOnBothClientServer TestSetUGIOnOnlyClient TestSetUGIOnOnlyServer TestHiveRemote TestAuthorizationPreEventListener TestRemoteHiveMetaStoreIpAddress TestMetaStoreEventListener TestRemoteUGIHiveMetaStoreIpAddress TestDefaultHiveMetastoreAuthorizationProvider TestExecDriver TestMetaStoreAuthorization TestHiveMetaStoreWithEnvironmentContext TestHBaseMinimrCliDriver TestHadoop20SAuthBridge TestHCatPartitionPublish [...]
-unitTests.exclude = TestHiveMetaStore TestSerDe TestBeeLineDriver TestHiveServer2Concurrency TestJdbcDriver2 TestHiveServer2Concurrency TestBeeLineDriver
-
-qFileTests = clientPositive miniMr clientNegative miniMrNegative
-
-qFileTest.clientPositive.driver = TestCliDriver
-qFileTest.clientPositive.directory = ql/src/test/queries/clientpositive
-qFileTest.clientPositive.batchSize = 15
-qFileTest.clientPositive.queryFilesProperty = qfile
-qFileTest.clientPositive.exclude = minimr
-qFileTest.clientPositive.groups.minimr = input16_cc.q scriptfile1.q bucket4.q bucketmapjoin6.q disable_merge_for_bucketing.q reduce_deduplicate.q smb_mapjoin_8.q join1.q groupby2.q bucketizedhiveinputformat.q bucketmapjoin7.q optrstat_groupby.q bucket_num_reducers.q
-
-qFileTest.miniMr.driver = TestMinimrCliDriver
-qFileTest.miniMr.directory = ql/src/test/queries/clientpositive
-qFileTest.miniMr.batchSize = 1000
-qFileTest.miniMr.queryFilesProperty = minimr.query.files
-qFileTest.miniMr.include = normal
-qFileTest.miniMr.isolate = flaky
-# normal are tests that run in minimr mode via build-common.xml
-qFileTest.miniMr.groups.normal = input16_cc.q scriptfile1.q bucket4.q bucketmapjoin6.q disable_merge_for_bucketing.q reduce_deduplicate.q smb_mapjoin_8.q join1.q groupby2.q bucketizedhiveinputformat.q bucketmapjoin7.q optrstat_groupby.q bucket_num_reducers.q
-qFileTest.miniMr.groups.flaky = bucketizedhiveinputformat.q
-
-qFileTest.miniMrNegative.driver = TestNegativeMinimrCliDriver
-qFileTest.miniMrNegative.directory = ql/src/test/queries/clientnegative
-qFileTest.miniMrNegative.batchSize = 1000
-qFileTest.miniMrNegative.queryFilesProperty = minimr.query.negative.files
-qFileTest.miniMrNegative.include = normal
-qFileTest.miniMrNegative.isolate = flaky
-# normal are tests that run in minimr mode via build-common.xml
-qFileTest.miniMrNegative.groups.normal = cluster_tasklog_retrieval.q minimr_broken_pipe.q mapreduce_stack_trace.q mapreduce_stack_trace_turnoff.q mapreduce_stack_trace_hadoop20.q mapreduce_stack_trace_turnoff_hadoop20.q
-qFileTest.miniMrNegative.groups.flaky = mapreduce_stack_trace_hadoop20.q
-
-qFileTest.clientNegative.driver = TestNegativeCliDriver
-qFileTest.clientNegative.directory = ql/src/test/queries/clientnegative
-qFileTest.clientNegative.batchSize = 1000
-qFileTest.clientNegative.queryFilesProperty = qfile
-qFileTest.clientNegative.exclude = miniMrNormal failing
-# stats_aggregator_error_1.q fails in both negative client and minimr mode
-qFileTest.clientNegative.groups.failing = stats_aggregator_error_1.q
-# normal are run via minimr
-qFileTest.clientNegative.groups.miniMrNormal = cluster_tasklog_retrieval.q minimr_broken_pipe.q mapreduce_stack_trace.q mapreduce_stack_trace_turnoff.q mapreduce_stack_trace_hadoop20.q mapreduce_stack_trace_turnoff_hadoop20.q
diff --git a/testutils/ptest2/conf/server-env.properties.example b/testutils/ptest2/conf/server-env.properties.example
deleted file mode 100644
index f23e9da..0000000
--- a/testutils/ptest2/conf/server-env.properties.example
+++ /dev/null
@@ -1,23 +0,0 @@
-#/**
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-#
-# This is an examle of properties needed by the PTest framework which are independent
-# of the per branch test profiles
-#
-javaHome = /usr/lib/jvm/java-8-openjdk-amd64
-javaHomeForTests = /usr/lib/jvm/java-8-openjdk-amd64
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
deleted file mode 100644
index 8e470ef..0000000
--- a/testutils/ptest2/pom.xml
+++ /dev/null
@@ -1,296 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-ptest</artifactId>
- <version>3.0</version>
- <packaging>war</packaging>
- <name>hive-ptest</name>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <log4j2.version>2.12.1</log4j2.version>
- <spring.framework.version>3.2.16.RELEASE</spring.framework.version>
- <jclouds.version>2.1.3</jclouds.version>
- <checkstyle.conf.dir>${basedir}/../../checkstyle/</checkstyle.conf.dir>
- <maven.checkstyle.plugin.version>2.12.1</maven.checkstyle.plugin.version>
- <jackson.version>2.12.0</jackson.version>
- </properties>
-
- <repositories>
- <repository>
- <id>spring-repository</id>
- <name>SpringSource Repository</name>
- <url>https://repo.springsource.org/milestone</url>
- </repository>
- <repository>
- <id>approvals-repository</id>
- <url>https://people.apache.org/~brock/approvals-maven-repository/</url>
- </repository>
- </repositories>
-
-
- <dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifactId -->
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.6</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- <version>1.2</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.9</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>18.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <version>${log4j2.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-web</artifactId>
- <version>${log4j2.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4j2.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.4</version>
- </dependency>
- <dependency>
- <groupId>org.apache.jclouds</groupId>
- <artifactId>jclouds-all</artifactId>
- <version>${jclouds.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.jclouds.driver</groupId>
- <artifactId>jclouds-sshj</artifactId>
- <version>${jclouds.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.jclouds.driver</groupId>
- <artifactId>jclouds-log4j</artifactId>
- <version>${jclouds.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- <version>1.7</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson</groupId>
- <artifactId>jackson-bom</artifactId>
- <version>${jackson.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.30</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-web</artifactId>
- <version>${spring.framework.version}</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-webmvc</artifactId>
- <version>${spring.framework.version}</version>
- </dependency>
- <dependency>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- <version>1.0.b2</version>
- </dependency>
-
- <!-- test -->
- <dependency>
- <groupId>approvaltests</groupId>
- <artifactId>ApprovalsJava</artifactId>
- <version>013</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>3.3.3</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <!-- plugins are always listed in sorted order by groupId, artifactId -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <configLocation>${checkstyle.conf.dir}/checkstyle.xml</configLocation>
- <propertyExpansion>basedir=${checkstyle.conf.dir}</propertyExpansion>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/lib/</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- <downloadJavadocs>true</downloadJavadocs>
- <wtpversion>2.0</wtpversion>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-report-plugin</artifactId>
- <version>2.15</version>
- <configuration>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- <version>2.3</version>
- <configuration>
- <attachClasses>true</attachClasses>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.tomcat.maven</groupId>
- <artifactId>tomcat7-maven-plugin</artifactId>
- <version>2.0</version>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <!-- plugins are always listed in sorted order by groupId, artifactId -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${maven.checkstyle.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>0.9</version>
- <executions>
- <execution>
- <id>verify.rat</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>check</goal>
- </goals>
- <configuration>
- <excludes>
- <exclude>*.patch/</exclude>
- <exclude>.idea/</exclude>
- <exclude>.git/</exclude>
- <exclude>.gitignore</exclude>
- <exclude>velocity.log</exclude>
- <exclude>README.md</exclude>
- <exclude>**/*.approved.txt</exclude>
- <exclude>**/.settings/*</exclude>
- <exclude>src/test/resources/</exclude>
- <exclude>**/.classpath</exclude>
- <exclude>**/.project</exclude>
- <exclude>**/target/**</exclude>
- </excludes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/Status.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/Status.java
deleted file mode 100644
index e9d0ee1..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/Status.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api;
-
-/**
- * Represents a generic status for a GenericResponse
- * in addition to the Status of a job.
- */
-public class Status {
- private Name name;
- private String message;
- public Status() {
-
- }
- public Status(Name name, String message) {
- this.name = name;
- this.message = message;
- }
- public Name getName() {
- return name;
- }
- public void setName(Name name) {
- this.name = name;
- }
- public String getMessage() {
- return message;
- }
- public void setMessage(String message) {
- this.message = message;
- }
-
- @Override
- public String toString() {
- return "Status [name=" + name + ", message=" + message + "]";
- }
-
- public static enum Name {
- ILLEGAL_ARGUMENT(),
- QUEUE_FULL(),
- INTERNAL_ERROR(),
- PENDING(),
- IN_PROGRESS(),
- FAILED(),
- OK();
- }
- public static void assertOK(Status status) {
- if(!isOK(status)) {
- throw new RuntimeException(status == null ? "Status is null" : status.toString());
- }
- }
- public static void assertOKOrFailed(Status status) {
- if(!(isOK(status) || isFailed(status))) {
- throw new RuntimeException(status == null ? "Status is null" : status.toString());
- }
- }
- public static boolean isInProgress(Status status) {
- return status != null && Name.IN_PROGRESS.equals(status.getName());
- }
- public static boolean isPending(Status status) {
- return status != null && Name.PENDING.equals(status.getName());
- }
- public static boolean isOK(Status status) {
- return status != null && Name.OK.equals(status.getName());
- }
- public static boolean isIllegalArgument(Status status) {
- return status != null && Name.ILLEGAL_ARGUMENT.equals(status.getName());
- }
- public static boolean isFailed(Status status) {
- return status != null && Name.FAILED.equals(status.getName());
- }
- public static Status illegalArgument() {
- return illegalArgument(null);
- }
- public static Status illegalArgument(String message) {
- return new Status(Name.ILLEGAL_ARGUMENT, message);
- }
- public static Status queueFull() {
- return new Status(Name.QUEUE_FULL, null);
- }
- public static Status internalError(String message) {
- return new Status(Name.INTERNAL_ERROR, message);
- }
- public static Status pending() {
- return new Status(Name.PENDING, null);
- }
- public static Status inProgress() {
- return new Status(Name.IN_PROGRESS, null);
- }
- public static Status failed(String message) {
- return new Status(Name.FAILED, message);
- }
- public static Status ok() {
- return new Status(Name.OK, null);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
deleted file mode 100644
index fd84169..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.client;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.io.IOUtils;
-import org.apache.hive.ptest.api.Status;
-import org.apache.hive.ptest.api.request.TestListRequest;
-import org.apache.hive.ptest.api.request.TestLogRequest;
-import org.apache.hive.ptest.api.request.TestStartRequest;
-import org.apache.hive.ptest.api.request.TestStatusRequest;
-import org.apache.hive.ptest.api.response.GenericResponse;
-import org.apache.hive.ptest.api.response.TestListResponse;
-import org.apache.hive.ptest.api.response.TestLogResponse;
-import org.apache.hive.ptest.api.response.TestStartResponse;
-import org.apache.hive.ptest.api.response.TestStatus;
-import org.apache.hive.ptest.api.response.TestStatusResponse;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Resources;
-
-/**
- * Quick and dirty REST client for the PTest server. It's not expected the scope
- * of this project will expand significantly therefore a simple REST client should suffice.
- */
-public class PTestClient {
- private static final ImmutableMap<Class<?>, EndPointResponsePair> REQUEST_TO_ENDPOINT =
- ImmutableMap.<Class<?>, EndPointResponsePair>builder()
- .put(TestStartRequest.class, new EndPointResponsePair("/testStart", TestStartResponse.class))
- .put(TestStatusRequest.class, new EndPointResponsePair("/testStatus", TestStatusResponse.class))
- .put(TestLogRequest.class, new EndPointResponsePair("/testLog", TestLogResponse.class))
- .put(TestListRequest.class, new EndPointResponsePair("/testList", TestListResponse.class))
- .build();
-
- private static final String HELP_LONG = "help";
- private static final String HELP_SHORT = "h";
- private static final String ENDPOINT = "endpoint";
- private static final String LOGS_ENDPOINT = "logsEndpoint";
- private static final String COMMAND = "command";
- private static final String PASSWORD = "password";
- private static final String PROFILE = "profile";
- private static final String PATCH = "patch";
- private static final String JIRA = "jira";
- private static final String OUTPUT_DIR = "outputDir";
- private static final String TEST_HANDLE = "testHandle";
- private static final String CLEAR_LIBRARY_CACHE = "clearLibraryCache";
- private static final int MAX_RETRIES = 10;
- private final String mApiEndPoint;
- private final String mLogsEndpoint;
- private final ObjectMapper mMapper;
- private final DefaultHttpClient mHttpClient;
- private final String testOutputDir;
-
- public PTestClient(String logsEndpoint, String apiEndPoint, String password, String testOutputDir)
- throws MalformedURLException {
- this.testOutputDir = testOutputDir;
- if (!Strings.isNullOrEmpty(testOutputDir)) {
- Preconditions.checkArgument(!Strings.isNullOrEmpty(logsEndpoint),
- "logsEndPoint must be specified if " + OUTPUT_DIR + " is specified");
- if (logsEndpoint.endsWith("/")) {
- this.mLogsEndpoint = logsEndpoint;
- } else {
- this.mLogsEndpoint = logsEndpoint + "/";
- }
- } else {
- this.mLogsEndpoint = null;
- }
- if(apiEndPoint.endsWith("/")) {
- this.mApiEndPoint = apiEndPoint + "api/v1";
- } else {
- this.mApiEndPoint = apiEndPoint + "/api/v1";
- }
- URL apiURL = new URL(mApiEndPoint);
- mMapper = new ObjectMapper();
- mHttpClient = new DefaultHttpClient();
- mHttpClient.getCredentialsProvider().setCredentials(
- new AuthScope(apiURL.getHost(), apiURL.getPort(), AuthScope.ANY_REALM),
- new UsernamePasswordCredentials("hive", password));
- }
- public boolean testStart(String profile, String testHandle,
- String jira, String patch, boolean clearLibraryCache)
- throws Exception {
- patch = Strings.nullToEmpty(patch).trim();
- if(!patch.isEmpty()) {
- byte[] bytes = Resources.toByteArray(new URL(patch));
- if(bytes.length == 0) {
- throw new IllegalArgumentException("Patch " + patch + " was zero bytes");
- }
- }
- TestStartRequest startRequest = new TestStartRequest(profile, testHandle, jira, patch, clearLibraryCache);
- post(startRequest, false);
- boolean result = false;
- try {
- result = testTailLog(testHandle);
- if(testOutputDir != null) {
- downloadTestResults(testHandle, testOutputDir);
- }
- } finally {
- System.out.println("\n\nLogs are located: " + mLogsEndpoint + testHandle + "\n\n");
- }
- return result;
- }
- public boolean testList()
- throws Exception {
- TestListRequest testListRequest = new TestListRequest();
- TestListResponse testListResponse = post(testListRequest, true);
- for(TestStatus testStatus : testListResponse.getEntries()) {
- System.out.println(testStatus);
- }
- return true;
- }
- public boolean testTailLog(String testHandle)
- throws Exception {
- testHandle = Strings.nullToEmpty(testHandle).trim();
- if(testHandle.isEmpty()) {
- throw new IllegalArgumentException("TestHandle is required");
- }
- TestStatusRequest statusRequest = new TestStatusRequest(testHandle);
- TestStatusResponse statusResponse;
- do {
- TimeUnit.SECONDS.sleep(5);
- statusResponse = post(statusRequest, true);
- } while(Status.isPending(statusResponse.getTestStatus().getStatus()));
- long offset = 0;
- do {
- long length = statusResponse.getTestStatus().getLogFileLength();
- if(length > offset) {
- offset = printLogs(testHandle, offset);
- } else {
- TimeUnit.SECONDS.sleep(5);
- }
- statusResponse = post(statusRequest, true);
- } while(Status.isInProgress(statusResponse.getTestStatus().getStatus()));
- while(offset < statusResponse.getTestStatus().getLogFileLength()) {
- offset = printLogs(testHandle, offset);
- }
- Status.assertOKOrFailed(statusResponse.getTestStatus().getStatus());
- return Status.isOK(statusResponse.getTestStatus().getStatus());
- }
- private void downloadTestResults(String testHandle, String testOutputDir)
- throws Exception {
- HttpGet request = new HttpGet(mLogsEndpoint + testHandle + "/test-results.tar.gz");
- FileOutputStream output = null;
- try {
- HttpResponse httpResponse = mHttpClient.execute(request);
- StatusLine statusLine = httpResponse.getStatusLine();
- if(statusLine.getStatusCode() != 200) {
- throw new RuntimeException(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
- }
- output = new FileOutputStream(new File(testOutputDir, "test-results.tar.gz"));
- IOUtils.copyLarge(httpResponse.getEntity().getContent(), output);
- output.flush();
- } finally {
- request.abort();
- if(output != null) {
- output.close();
- }
- }
- }
- private long printLogs(String testHandle, long offset)
- throws Exception {
- TestLogRequest logsRequest = new TestLogRequest(testHandle, offset, 64 * 1024);
- TestLogResponse logsResponse = post(logsRequest, true);
- System.out.print(logsResponse.getBody());
- return logsResponse.getOffset();
- }
- private <S extends GenericResponse> S post(Object payload, boolean agressiveRetry)
- throws Exception {
- EndPointResponsePair endPointResponse = Preconditions.
- checkNotNull(REQUEST_TO_ENDPOINT.get(payload.getClass()), payload.getClass().getName());
- HttpPost request = new HttpPost(mApiEndPoint + endPointResponse.getEndpoint());
- try {
- String payloadString = mMapper.writeValueAsString(payload);
- StringEntity params = new StringEntity(payloadString);
- request.addHeader("content-type", "application/json");
- request.setEntity(params);
- if(agressiveRetry) {
- mHttpClient.setHttpRequestRetryHandler(new PTestHttpRequestRetryHandler());
- }
- HttpResponse httpResponse = mHttpClient.execute(request);
- StatusLine statusLine = httpResponse.getStatusLine();
- if(statusLine.getStatusCode() != 200) {
- throw new IllegalStateException(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
- }
- String response = EntityUtils.toString(httpResponse.getEntity(), "UTF-8");
- @SuppressWarnings("unchecked")
- S result = (S)endPointResponse.
- getResponseClass().cast(mMapper.readValue(response, endPointResponse.getResponseClass()));
- Status.assertOK(result.getStatus());
- if(System.getProperty("DEBUG_PTEST_CLIENT") != null) {
- System.err.println("payload " + payloadString);
- if(result instanceof TestLogResponse) {
- System.err.println("response " + ((TestLogResponse)result).getOffset() + " " + ((TestLogResponse)result).getStatus());
- } else {
- System.err.println("response " + response);
- }
- }
- Thread.sleep(1000);
- return result;
- } finally {
- request.abort();
- }
- }
- public static class PTestHttpRequestRetryHandler implements HttpRequestRetryHandler {
- @Override
- public boolean retryRequest(IOException exception, int executionCount,
- HttpContext context) {
- System.err.println("LOCAL ERROR: " + exception.getMessage());
- exception.printStackTrace();
- if(executionCount > MAX_RETRIES) {
- return false;
- }
- try {
- Thread.sleep(30L * 1000L);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- return true;
- }
-
- }
- private static class EndPointResponsePair {
- final String endpoint;
- final Class<? extends GenericResponse> responseClass;
- public EndPointResponsePair(String endpoint,
- Class<? extends GenericResponse> responseClass) {
- this.endpoint = endpoint;
- this.responseClass = responseClass;
- }
- public String getEndpoint() {
- return endpoint;
- }
- public Class<? extends GenericResponse> getResponseClass() {
- return responseClass;
- }
- }
- private static void assertRequired(CommandLine commandLine, String[] requiredOptions) {
- for(String requiredOption : requiredOptions) {
- if(!commandLine.hasOption(requiredOption)) {
- throw new IllegalArgumentException(requiredOption + " is required");
- }
- }
- }
- public static void main(String[] args) throws Exception {
- CommandLineParser parser = new GnuParser();
- Options options = new Options();
- options.addOption(HELP_SHORT, HELP_LONG, false, "Display help text and exit");
- options.addOption(null, ENDPOINT, true, "Service to use. E.g. http://localhost/ (Required)");
- options.addOption(null, COMMAND, true, "Command: [testStart, testStop, testTailLog, testList] (Required)");
- options.addOption(null, PASSWORD, true, "Password for service. Any committer should know this otherwise as private@. (Required)");
- options.addOption(null, PROFILE, true, "Test profile such as trunk-mr1 or trunk-mr2 (Required for testStart)");
- options.addOption(null, PATCH, true, "URI to patch, must start with http(s):// (Optional for testStart)");
- options.addOption(null, JIRA, true, "JIRA to post the results to e.g.: HIVE-XXXX");
- options.addOption(null, TEST_HANDLE, true, "Server supplied test handle. (Required for testStop and testTailLog)");
- options.addOption(null, OUTPUT_DIR, true, "Directory to download and save test-results.tar.gz to. (Optional for testStart)");
- options.addOption(null, CLEAR_LIBRARY_CACHE, false, "Before starting the test, delete the ivy and maven directories (Optional for testStart)");
- options.addOption(null, LOGS_ENDPOINT, true, "URL to get the logs");
-
- CommandLine commandLine = parser.parse(options, args);
-
- if(commandLine.hasOption(HELP_SHORT)) {
- new HelpFormatter().printHelp(PTestClient.class.getName(), options, true);
- System.exit(0);
- }
- assertRequired(commandLine, new String[] {
- COMMAND,
- PASSWORD,
- ENDPOINT
- });
- PTestClient client = new PTestClient(commandLine.getOptionValue(LOGS_ENDPOINT), commandLine.getOptionValue(ENDPOINT),
- commandLine.getOptionValue(PASSWORD), commandLine.getOptionValue(OUTPUT_DIR));
- String command = commandLine.getOptionValue(COMMAND);
- boolean result;
- if("testStart".equalsIgnoreCase(command)) {
- assertRequired(commandLine, new String[] {
- PROFILE,
- TEST_HANDLE
- });
-
- result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE),
- commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH),
- commandLine.hasOption(CLEAR_LIBRARY_CACHE));
- } else if("testTailLog".equalsIgnoreCase(command)) {
- result = client.testTailLog(commandLine.getOptionValue(TEST_HANDLE));
- } else if("testList".equalsIgnoreCase(command)) {
- result = client.testList();
- } else {
- throw new IllegalArgumentException("Unknown " + COMMAND + ": " + command);
- }
- if(result) {
- System.exit(0);
- } else {
- System.exit(1);
- }
- }
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestListRequest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestListRequest.java
deleted file mode 100644
index 0df2985..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestListRequest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.request;
-
-public class TestListRequest {
- /**
- * Do not use. Required by Jackson.
- */
- private String dummy;
- public TestListRequest() {
-
- }
- public String getDummy() {
- return dummy;
- }
- public void setDummy(String dummy) {
- this.dummy = dummy;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestLogRequest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestLogRequest.java
deleted file mode 100644
index 6ea8fd9..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestLogRequest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.request;
-
-public class TestLogRequest {
- private String testHandle;
- private long offset;
- private long length;
-
- public TestLogRequest() {
-
- }
- public TestLogRequest(String testHandle, long offset, long length) {
- super();
- this.testHandle = testHandle;
- this.offset = offset;
- this.length = length;
- }
- public long getOffset() {
- return offset;
- }
- public void setOffset(long offset) {
- this.offset = offset;
- }
- public long getLength() {
- return length;
- }
- public void setLength(long length) {
- this.length = length;
- }
- public String getTestHandle() {
- return testHandle;
- }
- public void setTestHandle(String testHandle) {
- this.testHandle = testHandle;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java
deleted file mode 100644
index 8deed52..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.request;
-
-public class TestStartRequest {
- private String profile;
- private String testHandle;
- private String patchURL;
- private String jiraName;
- private boolean clearLibraryCache;
-
- public TestStartRequest() {
-
- }
- public TestStartRequest(String profile, String testHandle,
- String jiraName, String patchURL, boolean clearLibraryCache) {
- this.profile = profile;
- this.testHandle = testHandle;
- this.jiraName = jiraName;
- this.patchURL = patchURL;
- this.clearLibraryCache = clearLibraryCache;
- }
- public String getProfile() {
- return profile;
- }
- public void setProfile(String profile) {
- this.profile = profile;
- }
- public String getPatchURL() {
- return patchURL;
- }
- public void setPatchURL(String patchURL) {
- this.patchURL = patchURL;
- }
- public boolean isClearLibraryCache() {
- return clearLibraryCache;
- }
- public void setClearLibraryCache(boolean clearLibraryCache) {
- this.clearLibraryCache = clearLibraryCache;
- }
- public String getJiraName() {
- return jiraName;
- }
- public void setJiraName(String jiraName) {
- this.jiraName = jiraName;
- }
-
- public String getTestHandle() {
- return testHandle;
- }
- public void setTestHandle(String testHandle) {
- this.testHandle = testHandle;
- }
- @Override
- public String toString() {
- return "TestStartRequest [profile=" + profile + ", testHandle="
- + testHandle + ", patchURL=" + patchURL + ", jiraName=" + jiraName
- + ", clearLibraryCache=" + clearLibraryCache + "]";
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStatusRequest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStatusRequest.java
deleted file mode 100644
index 29bd26f..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStatusRequest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.request;
-
-public class TestStatusRequest {
- private String testHandle;
-
- public TestStatusRequest() {
-
- }
- public TestStatusRequest(String testHandle) {
- super();
- this.testHandle = testHandle;
- }
-
- public String getTestHandle() {
- return testHandle;
- }
-
- public void setTestHandle(String testHandle) {
- this.testHandle = testHandle;
- }
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStopRequest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStopRequest.java
deleted file mode 100644
index bd04908..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStopRequest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.request;
-
-public class TestStopRequest {
- private String testHandle;
-
- public TestStopRequest() {
-
- }
- public TestStopRequest(String testHandle) {
- super();
- this.testHandle = testHandle;
- }
-
- public String getTestHandle() {
- return testHandle;
- }
-
- public void setTestHandle(String testHandle) {
- this.testHandle = testHandle;
- }
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/GenericResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/GenericResponse.java
deleted file mode 100644
index 257988e..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/GenericResponse.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import org.apache.hive.ptest.api.Status;
-
-public interface GenericResponse {
- public Status getStatus();
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestListResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestListResponse.java
deleted file mode 100644
index 038b80a..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestListResponse.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestListResponse implements GenericResponse {
- private Status status;
- private List<TestStatus> entries;
-
- public TestListResponse() {
- this(null);
- }
- public TestListResponse(Status status) {
- this(status, Collections.<TestStatus>emptyList());
- }
- public TestListResponse(Status status, List<TestStatus> entries) {
- this.status = status;
- this.entries = entries;
- }
- @Override
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- public List<TestStatus> getEntries() {
- return entries;
- }
- public void setEntries(List<TestStatus> entries) {
- this.entries = entries;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestLogResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestLogResponse.java
deleted file mode 100644
index 8cccc17..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestLogResponse.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestLogResponse implements GenericResponse {
- private Status status;
- private String body;
- private long offset;
-
- public TestLogResponse() {
- this(null);
- }
- public TestLogResponse(Status status) {
- this(status, 0L, null);
- }
- public TestLogResponse(Status status, long offset, String body) {
- super();
- this.status = status;
- this.offset = offset;
- this.body = body;
- }
- @Override
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- public String getBody() {
- return body;
- }
- public void setBody(String body) {
- this.body = body;
- }
- public long getOffset() {
- return offset;
- }
- public void setOffset(long offset) {
- this.offset = offset;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStartResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStartResponse.java
deleted file mode 100644
index cc505cf..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStartResponse.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestStartResponse implements GenericResponse {
- private Status status;
-
- public TestStartResponse() {
- this(null);
- }
- public TestStartResponse(Status status) {
- super();
- this.status = status;
- }
- @Override
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- @Override
- public String toString() {
- return "StartResponse [status=" + status + "]";
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatus.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatus.java
deleted file mode 100644
index 7dfadb4..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatus.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestStatus {
- private String testHandle;
- private Status status;
- private long elapsedQueueTime;
- private long elapsedExecutionTime;
- private long logFileLength;
- public TestStatus(String testHandle, Status status, long elapsedQueueTime,
- long elapsedExecutionTime, long logFileLength) {
- super();
- this.testHandle = testHandle;
- this.status = status;
- this.elapsedQueueTime = elapsedQueueTime;
- this.elapsedExecutionTime = elapsedExecutionTime;
- this.logFileLength = logFileLength;
- }
- public TestStatus() {
-
- }
- public String getTestHandle() {
- return testHandle;
- }
- public void setTestHandle(String testHandle) {
- this.testHandle = testHandle;
- }
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- public long getElapsedQueueTime() {
- return elapsedQueueTime;
- }
- public void setElapsedQueueTime(long elapsedQueueTime) {
- this.elapsedQueueTime = elapsedQueueTime;
- }
- public long getElapsedExecutionTime() {
- return elapsedExecutionTime;
- }
- public void setElapsedExecutionTime(long elapsedExecutionTime) {
- this.elapsedExecutionTime = elapsedExecutionTime;
- }
- public long getLogFileLength() {
- return logFileLength;
- }
- public void setLogFileLength(long logFileLength) {
- this.logFileLength = logFileLength;
- }
- @Override
- public String toString() {
- return "TestStatus [testHandle=" + testHandle + ", status=" + status
- + ", elapsedQueueTime=" + toMinutes(elapsedQueueTime) + " minute(s), elapsedExecutionTime="
- + toMinutes(elapsedExecutionTime) + " minute(s), logFileLength=" + logFileLength + "]";
- }
- private long toMinutes(long value) {
- return TimeUnit.MINUTES.convert(value, TimeUnit.MILLISECONDS);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatusResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatusResponse.java
deleted file mode 100644
index 36795bb..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStatusResponse.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestStatusResponse implements GenericResponse {
- private Status status;
- private TestStatus testStatus;
-
- public TestStatusResponse() {
- this(null);
- }
- public TestStatusResponse(Status status) {
- this(status, null);
- }
- public TestStatusResponse(Status status, TestStatus testStatus) {
- this.status = status;
- this.testStatus = testStatus;
- }
- @Override
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- public TestStatus getTestStatus() {
- return testStatus;
- }
- public void setTestStatus(TestStatus testStatus) {
- this.testStatus = testStatus;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStopResponse.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStopResponse.java
deleted file mode 100644
index e0cc75e..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/response/TestStopResponse.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.response;
-
-import org.apache.hive.ptest.api.Status;
-
-public class TestStopResponse implements GenericResponse {
- private Status status;
-
- public TestStopResponse() {
- this(null);
- }
- public TestStopResponse(Status status) {
- this.status = status;
- }
- @Override
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
deleted file mode 100644
index 29789c7..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.server;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.hive.ptest.api.Status;
-import org.apache.hive.ptest.api.request.TestListRequest;
-import org.apache.hive.ptest.api.request.TestLogRequest;
-import org.apache.hive.ptest.api.request.TestStartRequest;
-import org.apache.hive.ptest.api.request.TestStopRequest;
-import org.apache.hive.ptest.api.response.TestListResponse;
-import org.apache.hive.ptest.api.response.TestLogResponse;
-import org.apache.hive.ptest.api.response.TestStartResponse;
-import org.apache.hive.ptest.api.response.TestStatus;
-import org.apache.hive.ptest.api.response.TestStatusResponse;
-import org.apache.hive.ptest.api.response.TestStopResponse;
-import org.apache.hive.ptest.execution.PTest;
-import org.apache.hive.ptest.execution.conf.Context;
-import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
-import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Controller;
-import org.springframework.validation.BindingResult;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-
-
-
-/**
- * Server interface of the ptest environment. Each request
- * is converted from JSON and each response is returned in JSON.
- */
-@Controller
-@RequestMapping(value = "/api/v1")
-public class ExecutionController {
- private static final long MAX_READ_SIZE = 1024L * 1024L;
- private static final String CONF_PROPERTY = "hive.ptest.execution.context.conf";
-
- private static final Logger LOG = LoggerFactory
- .getLogger(ExecutionController.class);
- private final ExecutionContextConfiguration mExecutionContextConfiguration;
- private final ExecutionContextProvider mExecutionContextProvider;
- private final Map<String, Test> mTests;
- private final BlockingQueue<Test> mTestQueue;
- private final TestExecutor mTestExecutor;
- private final File mGlobalLogDir;
- public ExecutionController()
- throws IOException {
- String executionContextConfigurationFile = System.getProperty(CONF_PROPERTY, "").trim();
- Preconditions.checkArgument(!executionContextConfigurationFile.isEmpty(), CONF_PROPERTY + " is required");
- LOG.info("Reading configuration from file: " + executionContextConfigurationFile);
- mExecutionContextConfiguration = ExecutionContextConfiguration.withContext(
- Context.fromFile(executionContextConfigurationFile)
- );
- LOG.info("ExecutionContext is [{}]", mExecutionContextConfiguration);
- mExecutionContextProvider = mExecutionContextConfiguration.getExecutionContextProvider();
- mTests = Collections.synchronizedMap(new LinkedHashMap<String, Test>() {
- private static final long serialVersionUID = 1L;
- @Override
- public boolean removeEldestEntry(Map.Entry<String, Test> entry) {
- Test testExecution = entry.getValue();
- File testOutputFile = testExecution.getOutputFile();
- return size() > 30 || (testOutputFile != null && !testOutputFile.isFile());
- }
- });
- mTestQueue = new ArrayBlockingQueue<Test>(5);
- mGlobalLogDir = new File(mExecutionContextConfiguration.getGlobalLogDirectory());
- mTestExecutor = new TestExecutor(mExecutionContextConfiguration, mExecutionContextProvider,
- mTestQueue, new PTest.Builder());
- mTestExecutor.setName("TestExecutor");
- mTestExecutor.setDaemon(true);
- mTestExecutor.start();
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- LOG.info("Shutdown hook called");
- try {
- mTestExecutor.shutdown();
- } catch (Exception e) {
- LOG.error("Error shutting down TestExecutor", e);
- }
- try {
- mExecutionContextProvider.close();
- } catch (Exception e) {
- LOG.error("Error shutting down ExecutionContextProvider", e);
- }
- }
- });
- }
-
- @RequestMapping(value="/testStart", method = RequestMethod.POST)
- public @ResponseBody TestStartResponse testStart(@RequestBody TestStartRequest startRequest,
- BindingResult result) {
- LOG.info("startRequest " + startRequest.toString());
- TestStartResponse startResponse = doStartTest(startRequest, result);
- LOG.info("startResponse " + startResponse.toString());
- return startResponse;
- }
-
- private TestStartResponse doStartTest(TestStartRequest startRequest, BindingResult result) {
- if(result.hasErrors() ||
- Strings.nullToEmpty(startRequest.getProfile()).trim().isEmpty() ||
- Strings.nullToEmpty(startRequest.getTestHandle()).trim().isEmpty() ||
- startRequest.getProfile().contains("/")) {
- return new TestStartResponse(Status.illegalArgument());
- }
- if(!assertTestHandleIsAvailable(startRequest.getTestHandle())) {
- return new TestStartResponse(Status.illegalArgument("Test handle " + startRequest.getTestHandle() + " already used"));
- }
- Test test = new Test(startRequest,
- Status.pending(), System.currentTimeMillis());
- if(mTestQueue.offer(test)) {
- mTests.put(startRequest.getTestHandle(), test);
- return new TestStartResponse(Status.ok());
- } else {
- return new TestStartResponse(Status.queueFull());
- }
- }
-
- @RequestMapping(value="/testStop", method = RequestMethod.POST)
- public @ResponseBody TestStopResponse testStop(@RequestBody TestStopRequest stopRequest,
- BindingResult result) {
- String testHandle = stopRequest.getTestHandle();
- Test test = mTests.get(testHandle);
- if(result.hasErrors() ||
- Strings.nullToEmpty(stopRequest.getTestHandle()).trim().isEmpty() ||
- test == null) {
- return new TestStopResponse(Status.illegalArgument());
- }
- test.setStopRequested(true);
- return new TestStopResponse(Status.ok());
- }
-
- @RequestMapping(value="/testStatus", method = RequestMethod.POST)
- public @ResponseBody TestStatusResponse testStatus(@RequestBody TestStopRequest stopRequest,
- BindingResult result) {
- String testHandle = stopRequest.getTestHandle();
- Test test = mTests.get(testHandle);
- if(result.hasErrors() ||
- Strings.nullToEmpty(stopRequest.getTestHandle()).trim().isEmpty() ||
- test == null) {
- return new TestStatusResponse(Status.illegalArgument());
- }
- return new TestStatusResponse(Status.ok(), test.toTestStatus());
- }
-
- @RequestMapping(value="/testLog", method = RequestMethod.POST)
- public @ResponseBody TestLogResponse testLog(@RequestBody TestLogRequest logsRequest,
- BindingResult result) {
- String testHandle = logsRequest.getTestHandle();
- Test testExecution = mTests.get(testHandle);
- if(result.hasErrors() ||
- Strings.nullToEmpty(logsRequest.getTestHandle()).trim().isEmpty() ||
- testExecution == null ||
- logsRequest.getLength() > MAX_READ_SIZE) {
- return new TestLogResponse(Status.illegalArgument());
- }
- File outputFile = testExecution.getOutputFile();
- if(outputFile == null ||
- logsRequest.getOffset() > outputFile.length()) {
- return new TestLogResponse(Status.illegalArgument());
- }
- RandomAccessFile fileHandle = null;
- try {
- fileHandle = new RandomAccessFile(outputFile, "r");
- long offset = logsRequest.getOffset();
- fileHandle.seek(offset);
- int readLength = 0;
- if(offset < fileHandle.length()) {
- readLength = (int)Math.min(fileHandle.length() - offset, logsRequest.getLength());
- }
- byte[] buffer = new byte[readLength];
- fileHandle.readFully(buffer);
- offset += readLength;
- return new TestLogResponse(Status.ok(), offset,
- new String(buffer, Charsets.UTF_8));
- } catch (IOException e) {
- LOG.info("Unexpected IO error reading " + testExecution.getOutputFile() , e);
- return new TestLogResponse(Status.internalError(e.getMessage()));
- } finally {
- if(fileHandle != null) {
- try {
- fileHandle.close();
- } catch (IOException e) {
- LOG.warn("Error closing " + outputFile, e);
- }
- }
- }
- }
-
- @RequestMapping(value="/testList", method = RequestMethod.POST)
- public @ResponseBody TestListResponse testList(@RequestBody TestListRequest request) {
- List<TestStatus> entries = Lists.newArrayList();
- synchronized (mTests) {
- for(String testHandle : mTests.keySet()) {
- Test test = mTests.get(testHandle);
- entries.add(test.toTestStatus());
- }
- }
- return new TestListResponse(Status.ok(), entries);
- }
- private synchronized boolean assertTestHandleIsAvailable(final String testHandle) {
- File testOutputDir = new File(mGlobalLogDir, testHandle);
- Preconditions.checkState(!testOutputDir.isFile(), "Output directory " + testOutputDir + " is file");
- return testOutputDir.mkdir();
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/Test.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/Test.java
deleted file mode 100644
index 7fbabd9..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/Test.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.server;
-
-import java.io.File;
-
-import org.apache.hive.ptest.api.Status;
-import org.apache.hive.ptest.api.request.TestStartRequest;
-import org.apache.hive.ptest.api.response.TestStatus;
-
-/**
- * Represents a test run in memory on the server.
- */
-class Test {
- private TestStartRequest startRequest;
- private Status status;
- private long enqueueTime;
- private long dequeueTime;
- private long executionStartTime;
- private long executionFinishTime;
- private File outputFile;
- private boolean stopRequested;
-
- Test(TestStartRequest startRequest,
- Status status, long enqueueTime) {
- super();
- this.startRequest = startRequest;
- this.status = status;
- this.enqueueTime = enqueueTime;
- }
- boolean isStopRequested() {
- return stopRequested;
- }
- void setStopRequested(boolean stopRequested) {
- this.stopRequested = stopRequested;
- }
- TestStartRequest getStartRequest() {
- return startRequest;
- }
- void setStartRequest(TestStartRequest startRequest) {
- this.startRequest = startRequest;
- }
- Status getStatus() {
- return status;
- }
- void setStatus(Status status) {
- this.status = status;
- }
- long getEnqueueTime() {
- return enqueueTime;
- }
- void setEnqueueTime(long enqueueTime) {
- this.enqueueTime = enqueueTime;
- }
- long getExecutionStartTime() {
- return executionStartTime;
- }
- void setExecutionStartTime(long executionStartTime) {
- this.executionStartTime = executionStartTime;
- }
- long getExecutionFinishTime() {
- return executionFinishTime;
- }
- void setExecutionFinishTime(long executionFinishTime) {
- this.executionFinishTime = executionFinishTime;
- }
- File getOutputFile() {
- return outputFile;
- }
- void setOutputFile(File outputFile) {
- this.outputFile = outputFile;
- }
- long getDequeueTime() {
- return dequeueTime;
- }
- void setDequeueTime(long dequeueTime) {
- this.dequeueTime = dequeueTime;
- }
- TestStatus toTestStatus() {
- long elapsedQueueTime;
- if(getDequeueTime() > 0) {
- elapsedQueueTime = getDequeueTime() - getEnqueueTime();
- } else {
- elapsedQueueTime = System.currentTimeMillis() - getEnqueueTime();
- }
- long elapsedExecutionTime = 0;
- if(getExecutionStartTime() > 0) {
- if(getExecutionFinishTime() > 0) {
- elapsedExecutionTime = getExecutionFinishTime() - getExecutionStartTime();
- } else {
- elapsedExecutionTime = System.currentTimeMillis() - getExecutionStartTime();
- }
- }
- long logFileLength = 0;
- if(getOutputFile() != null) {
- logFileLength = getOutputFile().length();
- }
- return new TestStatus(startRequest.getTestHandle(), getStatus(),
- elapsedQueueTime, elapsedExecutionTime, logFileLength);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
deleted file mode 100644
index d4d28c0..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.server;
-
-import java.io.File;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hive.ptest.api.Status;
-import org.apache.hive.ptest.api.request.TestStartRequest;
-import org.apache.hive.ptest.execution.Constants;
-import org.apache.hive.ptest.execution.Dirs;
-import org.apache.hive.ptest.execution.LocalCommandFactory;
-import org.apache.hive.ptest.execution.LogDirectoryCleaner;
-import org.apache.hive.ptest.execution.PTest;
-import org.apache.hive.ptest.execution.conf.Context;
-import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
-import org.apache.hive.ptest.execution.conf.TestConfiguration;
-import org.apache.hive.ptest.execution.context.CreateHostsFailedException;
-import org.apache.hive.ptest.execution.context.ExecutionContext;
-import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
-import org.apache.hive.ptest.execution.context.ServiceNotAvailableException;
-import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
-import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Executes parallel test in a single thread since the slaves
- * will be fully utilized by the test environment.
- */
-public class TestExecutor extends Thread {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestExecutor.class);
- private static final String SERVER_ENV_PROPERTIES = "hive.ptest.server.env.properties";
-
- private final ExecutionContextConfiguration mExecutionContextConfiguration;
- private final ExecutionContextProvider mExecutionContextProvider;
- private final BlockingQueue<Test> mTestQueue;
- private final PTest.Builder mPTestBuilder;
- private ExecutionContext mExecutionContext;
- private boolean execute;
-
- public TestExecutor(ExecutionContextConfiguration executionContextConfiguration,
- ExecutionContextProvider executionContextProvider,
- BlockingQueue<Test> testQueue, PTest.Builder pTestBuilder) {
- mExecutionContextConfiguration = executionContextConfiguration;
- mExecutionContextProvider = executionContextProvider;
- mTestQueue = testQueue;
- mPTestBuilder = pTestBuilder;
- execute = true;
- }
-
- @Override
- public void run() {
- while(execute) {
- Test test = null;
- PrintStream logStream = null;
- Logger logger = null;
- try {
- // start a log cleaner at the start of each test
- LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(mExecutionContextConfiguration.
- getGlobalLogDirectory()), mExecutionContextConfiguration.getMaxLogDirectoriesPerProfile());
- cleaner.setName("LogCleaner-" + mExecutionContextConfiguration.
- getGlobalLogDirectory());
- cleaner.setDaemon(true);
- cleaner.start();
- test = mTestQueue.poll(30, TimeUnit.MINUTES);
- if(!execute) {
- terminateExecutionContext();
- break;
- }
- if(test == null) {
- terminateExecutionContext();
- } else {
- test.setStatus(Status.inProgress());
- test.setDequeueTime(System.currentTimeMillis());
- if(mExecutionContext == null) {
- mExecutionContext = createExceutionContext();
- }
- test.setExecutionStartTime(System.currentTimeMillis());
- TestStartRequest startRequest = test.getStartRequest();
- String profile = startRequest.getProfile();
- File profileConfFile = new File(mExecutionContextConfiguration.getProfileDirectory(),
- String.format("%s.properties", profile));
- LOG.info("Attempting to run using profile file: {}", profileConfFile);
- if(!profileConfFile.isFile()) {
- test.setStatus(Status.illegalArgument(
- "Profile " + profile + " not found in directory " +
- mExecutionContextConfiguration.getProfileDirectory()));
- test.setExecutionFinishTime(System.currentTimeMillis());
- } else {
- File logDir = Dirs.create(new File(mExecutionContextConfiguration.
- getGlobalLogDirectory(), test.getStartRequest().getTestHandle()));
- File logFile = new File(logDir, "execution.txt");
- test.setOutputFile(logFile);
- logStream = new PrintStream(logFile);
- logger = new TestLogger(logStream, TestLogger.LEVEL.DEBUG);
-
- Context.ContextBuilder builder = new Context.ContextBuilder();
- builder.addPropertiesFile(profileConfFile);
-
- String environmentConfigurationFile = System.getProperty(SERVER_ENV_PROPERTIES, null);
- if (environmentConfigurationFile != null) {
- builder.addPropertiesFile(environmentConfigurationFile);
- }
-
- Context ctx = builder.build();
-
- TestConfiguration testConfiguration = TestConfiguration.withContext(ctx, logger);
- testConfiguration.setPatch(startRequest.getPatchURL());
- testConfiguration.setJiraName(startRequest.getJiraName());
- testConfiguration.setClearLibraryCache(startRequest.isClearLibraryCache());
- LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger);
- PTest ptest = mPTestBuilder.build(testConfiguration, mExecutionContext,
- test.getStartRequest().getTestHandle(), logDir,
- localCommandFactory, new SSHCommandExecutor(logger),
- new RSyncCommandExecutor(logger, mExecutionContextConfiguration.getMaxRsyncThreads(),
- localCommandFactory), logger);
- int result = ptest.run();
- if(result == Constants.EXIT_CODE_SUCCESS) {
- test.setStatus(Status.ok());
- } else {
- test.setStatus(Status.failed("Tests failed with exit code " + result));
- }
- logStream.flush();
- // if all drones where abandoned on a host, try replacing them.
- mExecutionContext.replaceBadHosts();
- }
- }
- } catch (Exception e) {
- LOG.error("Unxpected Error", e);
- if(test != null) {
- test.setStatus(Status.failed("Tests failed with exception " +
- e.getClass().getName() + ": " + e.getMessage()));
- if(logger != null) {
- String msg = "Error executing " + test.getStartRequest().getTestHandle();
- logger.error(msg, e);
- }
- }
- // if we died for any reason lets get a new set of hosts
- terminateExecutionContext();
- } finally {
- if(test != null) {
- test.setExecutionFinishTime(System.currentTimeMillis());
- }
- if(logStream != null) {
- logStream.flush();
- logStream.close();
- }
- }
- }
- }
-
- private void terminateExecutionContext() {
- if(mExecutionContext != null) {
- mExecutionContext.terminate();
- mExecutionContext = null;
- }
- }
- private ExecutionContext createExceutionContext()
- throws ServiceNotAvailableException, InterruptedException, CreateHostsFailedException {
- long start = System.currentTimeMillis();
- LOG.info("Attempting to create a new execution context");
- ExecutionContext result = mExecutionContextProvider.createExecutionContext();
- long elapsedTime = System.currentTimeMillis() - start;
- LOG.info("Context Creation time: " + TimeUnit.SECONDS.
- convert(elapsedTime, TimeUnit.MILLISECONDS) + " seconds");
- return result;
- }
-
- public void shutdown() {
- execute = false;
- this.interrupt();
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
deleted file mode 100644
index ca2c8a3..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.api.server;
-
-import java.io.PrintStream;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.slf4j.helpers.FormattingTuple;
-import org.slf4j.helpers.MarkerIgnoringBase;
-import org.slf4j.helpers.MessageFormatter;
-
-/**
- * Simple logger which allows each test to have it's own log file.
- */
-public class TestLogger extends MarkerIgnoringBase {
-
- private static final long serialVersionUID = -1711679924980202258L;
- private final LEVEL mLevel;
- private final PrintStream mLog;
- private SimpleDateFormat mDateFormatter;
-
- public TestLogger(PrintStream logFile, LEVEL level) {
- mLog = logFile;
- mLevel = level;
- mDateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
- }
-
- public static enum LEVEL {
- TRACE(1),
- DEBUG(2),
- INFO(3),
- WARN(4),
- ERROR(5);
- private int index;
- private LEVEL(int index) {
- this.index = index;
- }
- }
- @Override
- public boolean isTraceEnabled() {
- return mLevel.index >= LEVEL.TRACE.index;
- }
-
- @Override
- public void trace(String msg) {
- log(LEVEL.TRACE, msg, null);
- }
-
- @Override
- public void trace(String format, Object arg) {
- FormattingTuple ft = MessageFormatter.format(format, arg);
- log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void trace(String format, Object arg1, Object arg2) {
- FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
- log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void trace(String format, Object[] argArray) {
- FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
- log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void trace(String msg, Throwable t) {
- log(LEVEL.TRACE, msg, t);
- }
-
- @Override
- public boolean isDebugEnabled() {
- return mLevel.index >= LEVEL.DEBUG.index;
- }
-
- @Override
- public void debug(String msg) {
- log(LEVEL.DEBUG, msg, null);
- }
-
- @Override
- public void debug(String format, Object arg) {
- FormattingTuple ft = MessageFormatter.format(format, arg);
- log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void debug(String format, Object arg1, Object arg2) {
- FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
- log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void debug(String format, Object[] argArray) {
- FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
- log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void debug(String msg, Throwable t) {
- log(LEVEL.DEBUG, msg, t);
- }
-
- @Override
- public boolean isInfoEnabled() {
- return mLevel.index >= LEVEL.INFO.index;
- }
-
- @Override
- public void info(String msg) {
- log(LEVEL.INFO, msg, null);
- }
-
- @Override
- public void info(String format, Object arg) {
- FormattingTuple ft = MessageFormatter.format(format, arg);
- log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void info(String format, Object arg1, Object arg2) {
- FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
- log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void info(String format, Object[] argArray) {
- FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
- log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void info(String msg, Throwable t) {
- log(LEVEL.INFO, msg, t);
- }
-
- @Override
- public boolean isWarnEnabled() {
- return mLevel.index >= LEVEL.WARN.index;
- }
- @Override
- public void warn(String msg) {
- log(LEVEL.WARN, msg, null);
- }
-
- @Override
- public void warn(String format, Object arg) {
- FormattingTuple ft = MessageFormatter.format(format, arg);
- log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void warn(String format, Object arg1, Object arg2) {
- FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
- log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void warn(String format, Object[] argArray) {
- FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
- log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void warn(String msg, Throwable t) {
- log(LEVEL.WARN, msg, t);
- }
-
- @Override
- public boolean isErrorEnabled() {
- return mLevel.index >= LEVEL.ERROR.index;
- }
-
- @Override
- public void error(String msg) {
- log(LEVEL.ERROR, msg, null);
- }
-
- @Override
- public void error(String format, Object arg) {
- FormattingTuple ft = MessageFormatter.format(format, arg);
- log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void error(String format, Object arg1, Object arg2) {
- FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
- log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void error(String format, Object[] argArray) {
- FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
- log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
- }
-
- @Override
- public void error(String msg, Throwable t) {
- log(LEVEL.ERROR, msg, t);
- }
-
- private String getCaller() {
- StackTraceElement[] stack = new Exception().getStackTrace();
- if(stack.length > 3) {
- return getCallerShortName(stack[3]);
- }
- return "<unknown>";
- }
-
- private String getThreadName() {
- return Thread.currentThread().getName();
- }
-
- private String getCallerShortName(StackTraceElement frame) {
- String className = frame.getClassName();
- String methodName = frame.getMethodName();
- int lineNumber = frame.getLineNumber();
- int pos = className.lastIndexOf(".");
- if(pos > 0) {
- className = className.substring(pos + 1);
- }
- return String.format("%s.%s:%d", className, methodName, lineNumber);
- }
-
- private synchronized void log(LEVEL level, String msg, Throwable t) {
- if(level.index >= mLevel.index) {
- mLog.print(mDateFormatter.format(new Date()));
- mLog.print(" ");
- mLog.print(String.format("%5s", level.name()));
- mLog.print(" ");
- mLog.print("[");
- mLog.print(getThreadName());
- mLog.print("]");
- mLog.print(" ");
- mLog.print(getCaller());
- mLog.print(" ");
- mLog.print(msg);
- if(t != null) {
- mLog.print(" ");
- t.printStackTrace(mLog);
- }
- mLog.print("\n");
- mLog.flush();
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java
deleted file mode 100644
index f9edb56..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-public class AbortDroneException extends Exception {
- private static final long serialVersionUID = 6673699997331155666L;
- public AbortDroneException(String msg) {
- this(msg, null);
- }
- public AbortDroneException(String msg, Throwable throwable) {
- super(msg, throwable);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java
deleted file mode 100644
index 1848b1e..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-
-public class Constants {
-
- public static final int EXIT_CODE_EXCEPTION = -1;
- public static final int EXIT_CODE_SUCCESS = 0;
- public static final int EXIT_CODE_UNKNOWN = 255;
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java
deleted file mode 100644
index 78dbde9..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-
-public class Dirs {
-
- public static File createEmpty(File dir) throws IOException {
- FileUtils.deleteQuietly(dir);
- return create(dir);
- }
- public static File create(File dir) throws IOException {
- if(dir.isDirectory()) {
- return dir;
- }
- if(dir.mkdirs()) {
- return dir;
- }
- throw new IOException("Could not create " + dir);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java
deleted file mode 100644
index 2856124..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-
-public class Drone {
-
- private final String privateKey;
- private final String user;
- private final String host;
- private final int instance;
- private final String localDir;
-
- public Drone(String privateKey, String user, String host, int instance, String localDir) {
- this.privateKey = privateKey;
- this.user = user;
- this.host = host;
- this.instance = instance;
- this.localDir = localDir;
- }
-
- public String getPrivateKey() {
- return privateKey;
- }
-
- public String getUser() {
- return user;
- }
-
- public String getHost() {
- return host;
- }
-
- @Override
- public String toString() {
- return "Drone [user=" + user + ", host=" + host + ", instance=" + instance
- + "]";
- }
- public String getLocalDirectory() {
- return localDir;
- }
- public String getLocalLogDirectory() {
- return (new File(new File(localDir, getInstanceName()), "logs")).getAbsolutePath();
- }
- public String getInstanceName() {
- return String.format("%s-%s-%d", host, user, instance);
- }
- public int getInstance() {
- return instance;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
deleted file mode 100644
index 7ab98f6..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Joiner;
-import org.apache.hive.ptest.execution.conf.Host;
-import org.apache.hive.ptest.execution.conf.QFileTestBatch;
-import org.apache.hive.ptest.execution.conf.TestBatch;
-import org.apache.hive.ptest.execution.context.ExecutionContext;
-import org.slf4j.Logger;
-
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-
-public class ExecutionPhase extends Phase {
- private static final long FOUR_HOURS = 4L* 60L * 60L * 1000L;
- private final ExecutionContext executionContext;
- private final HostExecutorBuilder hostExecutorBuilder;
- private final File succeededLogDir;
- private final File failedLogDir;
- private final BlockingQueue<TestBatch> parallelWorkQueue;
- private final BlockingQueue<TestBatch> isolatedWorkQueue;
- private final Set<String> executedTests;
- private final Set<String> failedTests;
- private final Supplier<List<TestBatch>> testBatchSupplier;
- private final Set<TestBatch> failedTestResults;
-
- public ExecutionPhase(List<HostExecutor> hostExecutors, ExecutionContext executionContext,
- HostExecutorBuilder hostExecutorBuilder,
- LocalCommandFactory localCommandFactory,
- ImmutableMap<String, String> templateDefaults,
- File succeededLogDir, File failedLogDir, Supplier<List<TestBatch>> testBatchSupplier,
- Set<String> executedTests, Set<String> failedTests, Logger logger)
- throws IOException {
- super(hostExecutors, localCommandFactory, templateDefaults, logger);
- this.executionContext = executionContext;
- this.hostExecutorBuilder = hostExecutorBuilder;
- this.succeededLogDir = succeededLogDir;
- this.failedLogDir = failedLogDir;
- this.testBatchSupplier = testBatchSupplier;
- this.executedTests = executedTests;
- this.failedTests = failedTests;
- this.parallelWorkQueue = new LinkedBlockingQueue<TestBatch>();
- this.isolatedWorkQueue = new LinkedBlockingQueue<TestBatch>();
- this.failedTestResults = Collections.
- synchronizedSet(new HashSet<TestBatch>());
- }
- @Override
- public void execute() throws Throwable {
- long start = System.currentTimeMillis();
- List<TestBatch> testBatches = Lists.newArrayList();
- for(TestBatch batch : testBatchSupplier.get()) {
- testBatches.add(batch);
- if(batch.isParallel()) {
- parallelWorkQueue.add(batch);
- } else {
- isolatedWorkQueue.add(batch);
- }
- }
- logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
- isolatedWorkQueue.size());
- if (logger.isDebugEnabled()) {
- for (TestBatch testBatch : parallelWorkQueue) {
- logger.debug("PBatch: {}", testBatch);
- }
- for (TestBatch testBatch : isolatedWorkQueue) {
- logger.debug("IBatch: {}", testBatch);
- }
- }
- try {
- int expectedNumHosts = hostExecutors.size();
- initalizeHosts();
- resetPerfMetrics();
- do {
- replaceBadHosts(expectedNumHosts);
- List<ListenableFuture<Void>> results = Lists.newArrayList();
- for(HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
- results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
- }
- Futures.allAsList(results).get();
- } while(!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
- for(TestBatch batch : testBatches) {
- File batchLogDir;
- if(failedTestResults.contains(batch)) {
- batchLogDir = new File(failedLogDir, batch.getName());
- } else {
- batchLogDir = new File(succeededLogDir, batch.getName());
- }
- JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
- executedTests.addAll(parser.getAllExecutedTests());
- for (String failedTest : parser.getAllFailedTests()) {
- failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
- }
-
- // if the TEST*.xml was not generated or was corrupt, let someone know
- if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
- Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
- expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
- for (String testClass : expTestClasses) {
- StringBuilder messageBuilder = new StringBuilder();
- messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)")
- .append(" (batchId=").append(batch.getBatchId()).append(")");
- if (batch instanceof QFileTestBatch) {
- Collection<String> tests = ((QFileTestBatch)batch).getTests();
- if (tests.size() != 0) {
- messageBuilder.append("\n\t[");
- messageBuilder.append(Joiner.on(",").join(tests));
- messageBuilder.append("]");
- }
- }
- failedTests.add(messageBuilder.toString());
- }
- }
- }
- } finally {
- long elapsed = System.currentTimeMillis() - start;
- addAggregatePerfMetrics();
- logger.info("PERF: exec phase " +
- TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes");
- }
- }
-
- public static final String TOTAL_RSYNC_TIME = "TotalRsyncElapsedTime";
- private void addAggregatePerfMetrics() {
- long totalRsycTime = 0L;
- for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
- totalRsycTime += hostExecutor.getTotalRsyncTimeInMs();
- }
- addPerfMetric(TOTAL_RSYNC_TIME, totalRsycTime);
- }
-
- private void replaceBadHosts(int expectedNumHosts)
- throws Exception {
- Set<Host> goodHosts = Sets.newHashSet();
- for(HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
- if(hostExecutor.isBad()) {
- logger.info("Removing host during execution phase: " + hostExecutor.getHost());
- executionContext.addBadHost(hostExecutor.getHost());
- hostExecutors.remove(hostExecutor);
- } else {
- goodHosts.add(hostExecutor.getHost());
- }
- }
- long start = System.currentTimeMillis();
- while(hostExecutors.size() < expectedNumHosts) {
- if(System.currentTimeMillis() - start > FOUR_HOURS) {
- throw new RuntimeException("Waited over fours for hosts, still have only " +
- hostExecutors.size() + " hosts out of an expected " + expectedNumHosts);
- }
- logger.warn("Only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts
- + ", attempting to replace bad hosts");
- TimeUnit.MINUTES.sleep(1);
- executionContext.replaceBadHosts();
- for(Host host : executionContext.getHosts()) {
- if(!goodHosts.contains(host)) {
- HostExecutor hostExecutor = hostExecutorBuilder.build(host);
- initalizeHost(hostExecutor);
- if(hostExecutor.isBad()) {
- executionContext.addBadHost(hostExecutor.getHost());
- } else {
- logger.info("Adding new host during execution phase: " + host);
- hostExecutors.add(hostExecutor);
- }
- }
- }
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
deleted file mode 100644
index 8982afd..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.google.common.base.Stopwatch;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hive.ptest.execution.conf.Host;
-import org.apache.hive.ptest.execution.conf.TestBatch;
-import org.apache.hive.ptest.execution.ssh.RSyncCommand;
-import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
-import org.apache.hive.ptest.execution.ssh.RSyncResult;
-import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
-import org.apache.hive.ptest.execution.ssh.SSHCommand;
-import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
-import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
-import org.apache.hive.ptest.execution.ssh.SSHResult;
-import org.slf4j.Logger;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-
-class HostExecutor {
- private final Host mHost;
- private final List<Drone> mDrones;
- private final ListeningExecutorService mExecutor;
- private final SSHCommandExecutor mSSHCommandExecutor;
- private final RSyncCommandExecutor mRSyncCommandExecutor;
- private final ImmutableMap<String, String> mTemplateDefaults;
- private final Logger mLogger;
- private final File mLocalScratchDirectory;
- private final File mSuccessfulTestLogDir;
- private final File mFailedTestLogDir;
- private final long mNumPollSeconds;
- private final boolean fetchLogsForSuccessfulTests;
- private volatile boolean mShutdown;
- private int numParallelBatchesProcessed = 0;
- private int numIsolatedBatchesProcessed = 0;
- private AtomicLong totalElapsedTimeInRsync = new AtomicLong(0L);
-
- HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
- SSHCommandExecutor sshCommandExecutor,
- RSyncCommandExecutor rsyncCommandExecutor,
- ImmutableMap<String, String> templateDefaults, File scratchDir,
- File succeededLogDir, File failedLogDir, long numPollSeconds,
- boolean fetchLogsForSuccessfulTests, Logger logger) {
- List<Drone> drones = Lists.newArrayList();
- String[] localDirs = host.getLocalDirectories();
- for (int index = 0; index < host.getThreads(); index++) {
- drones.add(new Drone(privateKey, host.getUser(), host.getName(),
- index, localDirs[index % localDirs.length]));
- }
- mShutdown = false;
- mHost = host;
- mDrones = new CopyOnWriteArrayList<Drone>(drones);
- mExecutor = executor;
- mSSHCommandExecutor = sshCommandExecutor;
- mRSyncCommandExecutor = rsyncCommandExecutor;
- mTemplateDefaults = templateDefaults;
- mLocalScratchDirectory = scratchDir;
- mSuccessfulTestLogDir = succeededLogDir;
- mFailedTestLogDir = failedLogDir;
- mNumPollSeconds = numPollSeconds;
- this.fetchLogsForSuccessfulTests = fetchLogsForSuccessfulTests;
- mLogger = logger;
- }
-
- /**
- * @return failed tests
- */
- ListenableFuture<Void> submitTests(final BlockingQueue<TestBatch> parallelWorkQueue,
- final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults) {
- return mExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Stopwatch stopwatch = Stopwatch.createStarted();
- mLogger.info("Starting SubmitTests on host {}", getHost());
- try {
- executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults);
- } finally {
- stopwatch.stop();
- mLogger.info("Finishing submitTests on host: {}. ElapsedTime(ms)={}," +
- " NumParallelBatchesProcessed={}, NumIsolatedBatchesProcessed={}",
- new Object[]{getHost().toString(),
- stopwatch.elapsed(TimeUnit.MILLISECONDS), numParallelBatchesProcessed,
- numIsolatedBatchesProcessed});
- }
- return null;
- }
-
- });
- }
- @VisibleForTesting
- int remainingDrones() {
- return mDrones.size();
- }
- boolean isBad() {
- return mDrones.isEmpty();
- }
- Host getHost() {
- return mHost;
- }
- void shutdownNow() {
- this.mShutdown = true;
- }
- boolean isShutdown() {
- return mShutdown;
- }
-
- long getTotalRsyncTimeInMs() {
- return totalElapsedTimeInRsync.get();
- }
- /**
- * Executes parallel test until the parallel work queue is empty. Then
- * executes the isolated tests on the host. During each phase if a
- * AbortDroneException is thrown the drone is removed possibly
- * leaving this host with zero functioning drones. If all drones
- * are removed the host will be replaced before the next run.
- */
- private void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
- final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults)
- throws Exception {
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return;
- }
- mLogger.info("Starting parallel execution on " + mHost.getName());
- List<ListenableFuture<Void>> droneResults = Lists.newArrayList();
- for(final Drone drone : ImmutableList.copyOf(mDrones)) {
- droneResults.add(mExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- TestBatch batch = null;
- Stopwatch sw = Stopwatch.createUnstarted();
- try {
- do {
- batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return null;
- }
- if(batch != null) {
- numParallelBatchesProcessed++;
- sw.reset().start();
- try {
- if (!executeTestBatch(drone, batch, failedTestResults)) {
- failedTestResults.add(batch);
- }
- } finally {
- sw.stop();
- mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
- }
- }
- } while(!mShutdown && !parallelWorkQueue.isEmpty());
- } catch(AbortDroneException ex) {
- mDrones.remove(drone); // return value not checked due to concurrent access
- mLogger.error("Aborting drone during parallel execution", ex);
- if(batch != null) {
- Preconditions.checkState(parallelWorkQueue.add(batch),
- "Could not add batch to parallel queue " + batch);
- }
- }
- return null;
- }
- }));
- }
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return;
- }
- Futures.allAsList(droneResults).get();
- mLogger.info("Starting isolated execution on " + mHost.getName());
- for(Drone drone : ImmutableList.copyOf(mDrones)) {
- TestBatch batch = null;
- Stopwatch sw = Stopwatch.createUnstarted();
- try {
- do {
-
- batch = isolatedWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
- if(batch != null) {
- numIsolatedBatchesProcessed++;
- sw.reset().start();
- try {
- if (!executeTestBatch(drone, batch, failedTestResults)) {
- failedTestResults.add(batch);
- }
- } finally {
- sw.stop();
- mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(ms)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
- }
- }
- } while(!mShutdown && !isolatedWorkQueue.isEmpty());
- } catch(AbortDroneException ex) {
- mDrones.remove(drone); // return value not checked due to concurrent access
- mLogger.error("Aborting drone during isolated execution", ex);
- if(batch != null) {
- Preconditions.checkState(isolatedWorkQueue.add(batch),
- "Could not add batch to isolated queue " + batch);
- }
- }
- }
- }
- /**
- * Executes the test batch on the drone in question. If the command
- * exits with a status code of 255 throw an AbortDroneException.
- */
- private boolean executeTestBatch(Drone drone, TestBatch batch, Set<TestBatch> failedTestResults)
- throws IOException, SSHExecutionException, AbortDroneException {
- String scriptName = "hiveptest-" + batch.getName() + ".sh";
- File script = new File(mLocalScratchDirectory, scriptName);
- Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("batchName", batch.getName());
- templateVariables.put("testArguments", batch.getTestArguments());
- templateVariables.put("localDir", drone.getLocalDirectory());
- templateVariables.put("logDir", drone.getLocalLogDirectory());
- Preconditions.checkArgument(StringUtils.isNotBlank(batch.getTestModuleRelativeDir()));
- templateVariables.put("testModule", batch.getTestModuleRelativeDir());
- String command = Templates.getTemplateResult("bash $localDir/$instanceName/scratch/" + script.getName(),
- templateVariables);
- Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
- copyToDroneFromLocal(drone, script.getAbsolutePath(), "$localDir/$instanceName/scratch/" + scriptName);
- script.delete();
- Stopwatch sw = Stopwatch.createStarted();
- mLogger.info(drone + " executing " + batch + " with " + command);
- RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(), command, true).
- call();
- sw.stop();
- mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(ms)={}",
- new Object[]{batch.getName(),
- getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
- File batchLogDir = null;
- if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
- throw new AbortDroneException("Drone " + drone.toString() + " exited with " +
- Constants.EXIT_CODE_UNKNOWN + ": " + sshResult);
- }
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return false;
- }
- boolean result;
- if(sshResult.getExitCode() != 0 || sshResult.getException() != null) {
- result = false;
- batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
- } else {
- result = true;
- batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
- }
- copyFromDroneToLocal(drone, batchLogDir.getAbsolutePath(),
- drone.getLocalLogDirectory() + "/", fetchLogsForSuccessfulTests || !result);
- File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
- PrintWriter writer = new PrintWriter(logFile);
- writer.write(String.format("result = '%s'\n", sshResult.toString()));
- writer.write(String.format("output = '%s'\n", sshResult.getOutput()));
- if(sshResult.getException() != null) {
- sshResult.getException().printStackTrace(writer);
- }
- writer.close();
- return result;
- }
- /**
- * RSync from a single drone. If the command exits with a status of not 0
- * throw an AbortDroneException.
- */
- RSyncResult copyToDroneFromLocal(Drone drone, String localFile, String remoteFile)
- throws AbortDroneException, SSHExecutionException, IOException {
- Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("localDir", drone.getLocalDirectory());
- RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(),
- Templates.getTemplateResult(localFile, templateVariables),
- Templates.getTemplateResult(remoteFile, templateVariables),
- RSyncCommand.Type.FROM_LOCAL).call();
- if(result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
- throw new AbortDroneException("Drone " + drone + " exited with " +
- result.getExitCode() + ": " + result);
- }
- if(result.getException() != null || result.getExitCode() != 0) {
- throw new SSHExecutionException(result);
- }
- totalElapsedTimeInRsync.getAndAdd(result.getElapsedTimeInMs());
- return result;
- }
- /**
- * RSync file to all drones. If any drones exit with a status of not 0
- * they will be removed from use possibly leaving this host with zero
- * functioning drones.
- */
- ListenableFuture<List<ListenableFuture<RemoteCommandResult>>> rsyncFromLocalToRemoteInstances(final String localFile, final String remoteFile)
- throws InterruptedException, IOException {
- // the basic premise here is that we will rsync the directory to first working drone
- // then execute a local rsync on the node to the other drones. This keeps
- // us from executing tons of rsyncs on the master node conserving CPU
- return mExecutor.submit(new Callable<List<ListenableFuture<RemoteCommandResult>>>() {
- @Override
- public List<ListenableFuture<RemoteCommandResult>> call()
- throws Exception {
- List<Drone> drones = Lists.newArrayList(mDrones);
- List<ListenableFuture<RemoteCommandResult>> results = Lists.newArrayList();
- // local path doesn't depend on drone variables
- String resolvedLocalLocation = Files.simplifyPath(Templates.getTemplateResult(localFile, mTemplateDefaults));
- String remoteStagingLocation = null;
- for(final Drone drone : ImmutableList.copyOf(mDrones)) {
- Preconditions.checkState(remoteStagingLocation == null, "Remote staging location must be null at the start of the loop");
- final Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("localDir", drone.getLocalDirectory());
- String resolvedRemoteLocation = Files.simplifyPath(Templates.getTemplateResult(remoteFile, templateVariables));
- RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(),
- resolvedLocalLocation,
- resolvedRemoteLocation,
- RSyncCommand.Type.FROM_LOCAL).call();
- if(result.getExitCode() == Constants.EXIT_CODE_SUCCESS) {
- remoteStagingLocation = resolvedRemoteLocation;
- drones.remove(drone);
- mLogger.info("Successfully staged " + resolvedLocalLocation + " on " + remoteStagingLocation);
- break;
- } else {
- mDrones.remove(drone);
- mLogger.error("Aborting drone during rsync",
- new AbortDroneException("Drone " + drone + " exited with "
- + result.getExitCode() + ": " + result));
- }
- }
- if(remoteStagingLocation == null) {
- Preconditions.checkState(mDrones.isEmpty(), "If remote staging location is not set all drones should be bad");
- mLogger.warn("Unable to stage directory on remote host, all drones must be bad");
- } else {
- String name = (new File(resolvedLocalLocation)).getName();
- remoteStagingLocation = Files.simplifyPath(remoteStagingLocation + "/" + name);
- results.addAll(execInstances(drones, "rsync -qaPe --delete --delete-during --force " + remoteStagingLocation + " " + remoteFile));
- }
- return results;
- }
- });
- }
- RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFile, boolean fetchAllLogs)
- throws SSHExecutionException, IOException {
- Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("localDir", drone.getLocalDirectory());
- RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(),
- Templates.getTemplateResult(localFile, templateVariables),
- Templates.getTemplateResult(remoteFile, templateVariables),
- fetchAllLogs ? RSyncCommand.Type.TO_LOCAL : RSyncCommand.Type.TO_LOCAL_NON_RECURSIVE).call();
- if(result.getException() != null || result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
- throw new SSHExecutionException(result);
- }
- totalElapsedTimeInRsync.getAndAdd(result.getElapsedTimeInMs());
- return result;
- }
- /**
- * Execute command on at least one drone. The method will retry when the command
- * exits with a status code of 255 until all drones have been utilized, possibly
- * excluding the host from future use.
- */
- ListenableFuture<SSHResult> execIgnoreAllErrors(final String cmd)
- throws Exception {
- return exec(cmd, false);
- }
- /**
- * Execute command on at least one drone. The method will retry when the command
- * exits with a status code of 255 until all drones have been utilized, possibly
- * excluding the host from future use.
- */
- ListenableFuture<SSHResult> exec(final String cmd)
- throws Exception {
- return exec(cmd, true);
- }
-
- private ListenableFuture<SSHResult> exec(final String cmd, final boolean reportErrors)
- throws Exception {
- return mExecutor.submit(new Callable<SSHResult>() {
- @Override
- public SSHResult call() throws Exception {
- for(final Drone drone : ImmutableList.copyOf(mDrones)) {
- Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("localDir", drone.getLocalDirectory());
- String command = Templates.getTemplateResult(cmd, templateVariables);
- SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(), command, reportErrors).call();
- if(reportErrors && result.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
- mDrones.remove(drone); // return value not checked due to concurrent access
- mLogger.error("Aborting drone during exec " + command,
- new AbortDroneException("Drone " + drone + " exited with "
- + Constants.EXIT_CODE_UNKNOWN + ": " + result));
- } else {
- return result;
- }
- }
- return null;
- }
- });
-
- }
- List<ListenableFuture<RemoteCommandResult>> execInstances(final String cmd)
- throws InterruptedException, IOException {
- return execInstances(mDrones, cmd);
- }
- private List<ListenableFuture<RemoteCommandResult>> execInstances(List<Drone> drones, final String cmd)
- throws InterruptedException, IOException {
- List<ListenableFuture<RemoteCommandResult>> result = Lists.newArrayList();
- for(final Drone drone : ImmutableList.copyOf(drones)) {
- result.add(mExecutor.submit(new Callable<RemoteCommandResult>() {
- @Override
- public RemoteCommandResult call() throws Exception {
- Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
- templateVariables.put("instanceName", drone.getInstanceName());
- templateVariables.put("localDir", drone.getLocalDirectory());
- String command = Templates.getTemplateResult(cmd, templateVariables);
- SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
- drone.getHost(), drone.getInstance(), command, true).call();
- if(result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
- mDrones.remove(drone); // return value not checked due to concurrent access
- mLogger.error("Aborting drone during exec " + command,
- new AbortDroneException("Drone " + drone + " exited with "
- + result.getExitCode() + ": " + result));
- return null;
- } else {
- return result;
- }
- }
- }));
- }
- return result;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutorBuilder.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutorBuilder.java
deleted file mode 100644
index 3603752..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutorBuilder.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import org.apache.hive.ptest.execution.conf.Host;
-
-public interface HostExecutorBuilder {
- public HostExecutor build(Host host);
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java
deleted file mode 100644
index 8eeb061..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java
+++ /dev/null
@@ -1,464 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.*;
-
-import com.google.common.collect.Sets;
-import org.apache.commons.cli.*;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.hive.ptest.api.server.TestLogger;
-import org.apache.hive.ptest.execution.conf.Context;
-import org.apache.hive.ptest.execution.conf.TestConfiguration;
-import org.apache.http.HttpException;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.auth.AuthScheme;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.AuthState;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.ClientContext;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.protocol.BasicHttpContext;
-import org.apache.http.protocol.ExecutionContext;
-import org.apache.http.protocol.HttpContext;
-import org.slf4j.Logger;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-class JIRAService {
- static final int MAX_MESSAGES = 200;
- static final String TRIMMED_MESSAGE = "**** This message was trimmed, see log for full details ****";
- private final Logger mLogger;
- private final String mName;
- private final String mBuildTag;
- private final String mPatch;
- private final String mUrl;
- private final String mUser;
- private final String mPassword;
- private final String mJenkinsURL;
- private final String mLogsURL;
-
- private static final String OPT_HELP_SHORT = "h";
- private static final String OPT_HELP_LONG = "help";
- private static final String OPT_USER_SHORT = "u";
- private static final String OPT_USER_LONG = "user";
- private static final String OPT_PASS_SHORT = "p";
- private static final String OPT_PASS_LONG = "password";
- private static final String OPT_FILE_SHORT = "f";
- private static final String OPT_FILE_LONG = "file";
-
- public JIRAService(Logger logger, TestConfiguration configuration, String buildTag) {
- mLogger = logger;
- mName = configuration.getJiraName();
- mBuildTag = buildTag;
- mPatch = configuration.getPatch();
- mUrl = configuration.getJiraUrl();
- mUser = configuration.getJiraUser();
- mPassword = configuration.getJiraPassword();
- mJenkinsURL = configuration.getJenkinsURL();
- mLogsURL = configuration.getLogsURL();
- }
-
- void postComment(boolean error, int numTestsExecuted, SortedSet<String> failedTests,
- List<String> messages) {
- postComment(error, numTestsExecuted, failedTests, messages, new HashSet<String>());
- }
-
- void postComment(boolean error, int numTestsExecuted, SortedSet<String> failedTests,
- List<String> messages, Set<String> addedTests) {
- String comments = generateComments(error, numTestsExecuted, failedTests, messages, addedTests);
- publishComments(comments);
- }
-
- @VisibleForTesting
- String generateComments(boolean error, int numTestsExecuted, SortedSet<String> failedTests,
- List<String> messages, Set<String> addedTests) {
- BuildInfo buildInfo = formatBuildTag(mBuildTag);
- String buildTagForLogs = formatBuildTagForLogs(mBuildTag);
- List<String> comments = Lists.newArrayList();
- comments.add("");
- comments.add("");
- if (!mPatch.isEmpty()) {
- comments.add("Here are the results of testing the latest attachment:");
- comments.add(mPatch);
- }
- comments.add("");
- if (error && numTestsExecuted == 0) {
- comments.add(formatError("-1 due to build exiting with an error"));
- } else {
- if (addedTests.size() > 0) {
- comments.add(formatSuccess("+1 due to " + addedTests.size() + " test(s) being added or modified."));
- } else {
- comments.add(formatError("-1 due to no test(s) being added or modified."));
- }
- comments.add("");
- if (numTestsExecuted == 0) {
- comments.add(formatError("-1 due to no tests executed"));
- } else {
- if (failedTests.isEmpty()) {
- comments.add(formatSuccess("+1 due to " + numTestsExecuted + " tests passed"));
- } else {
- comments.add(formatError("-1 due to " + failedTests.size()
- + " failed/errored test(s), " + numTestsExecuted + " tests executed"));
- comments.add("*Failed tests:*");
- comments.add("{noformat}");
- comments.addAll(failedTests);
- comments.add("{noformat}");
- }
- }
- }
- comments.add("");
- comments.add("Test results: " + mJenkinsURL + "/" +
- buildInfo.getFormattedBuildTag() + "/testReport");
- comments.add("Console output: " + mJenkinsURL + "/" +
- buildInfo.getFormattedBuildTag() + "/console");
- comments.add("Test logs: " + mLogsURL + buildTagForLogs);
- comments.add("");
- if (!messages.isEmpty()) {
- comments.add("Messages:");
- comments.add("{noformat}");
- comments.addAll(trimMessages(messages));
- comments.add("{noformat}");
- comments.add("");
- }
- comments.add("This message is automatically generated.");
- String attachmentId = parseAttachementId(mPatch);
- comments.add("");
- comments.add("ATTACHMENT ID: " + attachmentId +
- " - " + buildInfo.getBuildName());
- mLogger.info("Comment: " + Joiner.on("\n").join(comments));
- return Joiner.on("\n").join(comments);
- }
-
- void publishComments(String comments) {
- DefaultHttpClient httpClient = new DefaultHttpClient();
- try {
- String url = String.format("%s/rest/api/2/issue/%s/comment", mUrl, mName);
- URL apiURL = new URL(mUrl);
- httpClient.getCredentialsProvider()
- .setCredentials(
- new AuthScope(apiURL.getHost(), apiURL.getPort(),
- AuthScope.ANY_REALM),
- new UsernamePasswordCredentials(mUser, mPassword));
- BasicHttpContext localcontext = new BasicHttpContext();
- localcontext.setAttribute("preemptive-auth", new BasicScheme());
- httpClient.addRequestInterceptor(new PreemptiveAuth(), 0);
- HttpPost request = new HttpPost(url);
- ObjectMapper mapper = new ObjectMapper();
- StringEntity params = new StringEntity(mapper.writeValueAsString(new Body(comments)));
- request.addHeader("Content-Type", "application/json");
- request.setEntity(params);
- HttpResponse httpResponse = httpClient.execute(request, localcontext);
- StatusLine statusLine = httpResponse.getStatusLine();
- if (statusLine.getStatusCode() != 201) {
- throw new RuntimeException(statusLine.getStatusCode() + " "
- + statusLine.getReasonPhrase());
- }
- mLogger.info("JIRA Response Metadata: " + httpResponse);
- } catch (Exception e) {
- mLogger.error("Encountered error attempting to post comment to " + mName,
- e);
- } finally {
- httpClient.getConnectionManager().shutdown();
- }
- }
-
- static List<String> trimMessages(List<String> messages) {
- int size = messages.size();
- if (size > MAX_MESSAGES) {
- messages = messages.subList(size - MAX_MESSAGES, size);
- messages.add(0, TRIMMED_MESSAGE);
- }
- return messages;
- }
-
- @SuppressWarnings("unused")
- private static class Body {
- private String body;
-
- public Body() {
-
- }
-
- public Body(String body) {
- this.body = body;
- }
-
- public String getBody() {
- return body;
- }
-
- public void setBody(String body) {
- this.body = body;
- }
- }
-
-
- public static class BuildInfo {
- private String buildName;
- private String formattedBuildTag;
-
- public BuildInfo(String buildName, String formattedBuildTag) {
- this.buildName = buildName;
- this.formattedBuildTag = formattedBuildTag;
- }
-
- public String getBuildName() {
- return buildName;
- }
-
- public String getFormattedBuildTag() {
- return formattedBuildTag;
- }
- }
-
- /**
- * Hive-Build-123 to Hive-Build/123
- */
- @VisibleForTesting
- static BuildInfo formatBuildTag(String buildTag) {
- if (buildTag.contains("-")) {
- int lastDashIndex = buildTag.lastIndexOf("-");
- String buildName = buildTag.substring(0, lastDashIndex);
- String buildId = buildTag.substring(lastDashIndex + 1);
- String formattedBuildTag = buildName + "/" + buildId;
- return new BuildInfo(buildName, formattedBuildTag);
- }
- throw new IllegalArgumentException("Build tag '" + buildTag + "' must contain a -");
- }
-
- static String formatBuildTagForLogs(String buildTag) {
- if (buildTag.endsWith("/")) {
- return buildTag;
- } else {
- return buildTag + "/";
- }
- }
-
- private static String formatError(String msg) {
- return String.format("{color:red}ERROR:{color} %s", msg);
- }
-
- private static String formatSuccess(String msg) {
- return String.format("{color:green}SUCCESS:{color} %s", msg);
- }
-
- static class PreemptiveAuth implements HttpRequestInterceptor {
-
- public void process(final HttpRequest request, final HttpContext context)
- throws HttpException, IOException {
- AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
- if (authState.getAuthScheme() == null) {
- AuthScheme authScheme = (AuthScheme) context.getAttribute("preemptive-auth");
- CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(ClientContext.CREDS_PROVIDER);
- HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
- if (authScheme != null) {
- Credentials creds = credsProvider.getCredentials(new AuthScope(
- targetHost.getHostName(), targetHost.getPort()));
- if (creds == null) {
- throw new HttpException(
- "No credentials for preemptive authentication");
- }
- authState.update(authScheme, creds);
- }
- }
- }
- }
-
- private static String parseAttachementId(String patch) {
- if (patch == null) {
- return "";
- }
- String result = FilenameUtils.getPathNoEndSeparator(patch.trim());
- if (result == null) {
- return "";
- }
- result = FilenameUtils.getName(result.trim());
- if (result == null) {
- return "";
- }
- return result.trim();
- }
-
- private static void assertRequired(CommandLine commandLine, String[] requiredOptions) throws IllegalArgumentException {
- for (String requiredOption : requiredOptions) {
- if (!commandLine.hasOption(requiredOption)) {
- throw new IllegalArgumentException("--" + requiredOption + " is required");
- }
- }
- }
-
- private static final String FIELD_BUILD_STATUS = "buildStatus";
- private static final String FIELD_BUILD_TAG = "buildTag";
- private static final String FIELD_LOGS_URL = "logsURL";
- private static final String FIELD_JENKINS_URL = "jenkinsURL";
- private static final String FIELD_PATCH_URL = "patchUrl";
- private static final String FIELD_JIRA_NAME = "jiraName";
- private static final String FIELD_JIRA_URL = "jiraUrl";
- private static final String FIELD_REPO = "repository";
- private static final String FIELD_REPO_NAME = "repositoryName";
- private static final String FIELD_REPO_TYPE = "repositoryType";
- private static final String FIELD_REPO_BRANCH = "branch";
- private static final String FIELD_NUM_TESTS_EXECUTED = "numTestsExecuted";
- private static final String FIELD_FAILED_TESTS = "failedTests";
- private static final String FIELD_MESSAGES = "messages";
- private static final String FIELD_JIRA_USER = "jiraUser";
- private static final String FIELD_JIRA_PASS = "jiraPassword";
-
- private static Map<String, Class> supportedJsonFields = new HashMap<String, Class>() {
- {
- put(FIELD_BUILD_STATUS, Integer.class);
- put(FIELD_BUILD_TAG, String.class);
- put(FIELD_LOGS_URL, String.class);
- put(FIELD_JENKINS_URL, String.class);
- put(FIELD_PATCH_URL, String.class);
- put(FIELD_JIRA_NAME, String.class);
- put(FIELD_JIRA_URL, String.class);
- put(FIELD_REPO, String.class);
- put(FIELD_REPO_NAME, String.class);
- put(FIELD_REPO_TYPE, String.class);
- put(FIELD_REPO_BRANCH, String.class);
- put(FIELD_NUM_TESTS_EXECUTED, Integer.class);
- put(FIELD_FAILED_TESTS, SortedSet.class);
- put(FIELD_MESSAGES, List.class);
- }
- };
-
- private static Map<String, Object> parseJsonFile(String jsonFile) throws IOException {
- JsonFactory jsonFactory = new JsonFactory();
- JsonParser jsonParser = jsonFactory.createJsonParser(new File(jsonFile));
- Map<String, Object> values = new HashMap<String, Object>();
-
- while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
- String fieldName = jsonParser.getCurrentName();
- if (supportedJsonFields.containsKey(fieldName)) {
- jsonParser.nextToken();
-
- Class clazz = supportedJsonFields.get(fieldName);
- if (clazz == String.class) {
- values.put(fieldName, jsonParser.getText());
- } else if (clazz == Integer.class) {
- values.put(fieldName, Integer.valueOf(jsonParser.getText()));
- } else if (clazz == SortedSet.class) {
- SortedSet<String> failedTests = new TreeSet<String>();
- while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
- failedTests.add(jsonParser.getText());
- }
-
- values.put(fieldName, failedTests);
- } else if (clazz == List.class) {
- List<String> messages = new ArrayList<String>();
- while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
- messages.add(jsonParser.getText());
- }
-
- values.put(fieldName, messages);
- }
- }
- }
-
- jsonParser.close();
- return values;
- }
-
- private static CommandLine parseCommandLine(String[] args) throws ParseException {
- CommandLineParser parser = new GnuParser();
-
- Options options = new Options();
- options.addOption(OPT_HELP_SHORT, OPT_HELP_LONG, false, "Display help text and exit");
- options.addOption(OPT_USER_SHORT, OPT_USER_LONG, true, "Jira username.");
- options.addOption(OPT_PASS_SHORT, OPT_PASS_LONG, true, "Jira password.");
- options.addOption(OPT_FILE_SHORT, OPT_FILE_LONG, true, "Pathname to file (JSON format) that will be post as Jira comment.");
-
- CommandLine cmd = parser.parse(options, args);
-
- // If help option is requested, then display help and exit
- if (cmd.hasOption(OPT_HELP_LONG)) {
- new HelpFormatter().printHelp(JIRAService.class.getName(), options, true);
- return null;
- }
-
- assertRequired(cmd, new String[]{
- OPT_USER_LONG,
- OPT_PASS_LONG,
- OPT_FILE_LONG
- });
-
- return cmd;
- }
-
- public static void main(String[] args) throws Exception {
- CommandLine cmd = null;
-
- try {
- cmd = parseCommandLine(args);
- } catch (ParseException e) {
- System.out.println("Error parsing command arguments: " + e.getMessage());
- System.exit(1);
- }
-
- // If null is returned, then help message was displayed in parseCommandLine method
- if (cmd == null) {
- System.exit(0);
- }
-
- Map<String, Object> jsonValues = parseJsonFile(cmd.getOptionValue(OPT_FILE_LONG));
-
- Map<String, String> context = Maps.newHashMap();
- context.put(FIELD_JIRA_URL, (String) jsonValues.get(FIELD_JIRA_URL));
- context.put(FIELD_JIRA_USER, cmd.getOptionValue(OPT_USER_LONG));
- context.put(FIELD_JIRA_PASS, cmd.getOptionValue(OPT_PASS_LONG));
- context.put(FIELD_LOGS_URL, (String) jsonValues.get(FIELD_LOGS_URL));
- context.put(FIELD_REPO, (String) jsonValues.get(FIELD_REPO));
- context.put(FIELD_REPO_NAME, (String) jsonValues.get(FIELD_REPO_NAME));
- context.put(FIELD_REPO_TYPE, (String) jsonValues.get(FIELD_REPO_TYPE));
- context.put(FIELD_REPO_BRANCH, (String) jsonValues.get(FIELD_REPO_BRANCH));
- context.put(FIELD_JENKINS_URL, (String) jsonValues.get(FIELD_JENKINS_URL));
-
- TestLogger logger = new TestLogger(System.err, TestLogger.LEVEL.TRACE);
- TestConfiguration configuration = new TestConfiguration(new Context(context), logger);
- configuration.setJiraName((String) jsonValues.get(FIELD_JIRA_NAME));
- configuration.setPatch((String) jsonValues.get(FIELD_PATCH_URL));
-
- JIRAService service = new JIRAService(logger, configuration, (String) jsonValues.get(FIELD_BUILD_TAG));
- List<String> messages = (List) jsonValues.get(FIELD_MESSAGES);
- SortedSet<String> failedTests = (SortedSet) jsonValues.get(FIELD_FAILED_TESTS);
- boolean error = (Integer) jsonValues.get(FIELD_BUILD_STATUS) == 0 ? false : true;
- service.postComment(error, (Integer) jsonValues.get(FIELD_NUM_TESTS_EXECUTED), failedTests, messages);
- }
-}
\ No newline at end of file
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java
deleted file mode 100644
index 8fe56fe..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-
-import org.slf4j.Logger;
-import org.xml.sax.Attributes;
-import org.xml.sax.InputSource;
-import org.xml.sax.helpers.DefaultHandler;
-
-import com.google.common.collect.Sets;
-
-
-public class JUnitReportParser {
-
- private final File directory;
- private final Logger logger;
- private final Set<String> executedTests;
- private final Set<String> failedTests;
- private final Set<String> testClassesWithAvailableReports;
- private final Set<File> testOutputFiles;
- private boolean parsed;
- public JUnitReportParser(Logger logger, File directory) throws Exception {
- this.logger = logger;
- this.directory = directory;
- executedTests = Sets.newHashSet();
- failedTests = Sets.newHashSet();
- testOutputFiles = Sets.newHashSet();
- testClassesWithAvailableReports = Sets.newHashSet();
- parsed = false;
- }
-
- public Set<String> getAllExecutedTests() {
- parseIfRequired();
- return executedTests;
- }
-
- public Set<String> getAllFailedTests() {
- parseIfRequired();
- return failedTests;
- }
-
- public Set<String> getTestClassesWithReportAvailable() {
- return Collections.unmodifiableSet(testClassesWithAvailableReports);
- }
-
- public int getNumAttemptedTests() {
- parseIfRequired();
- return getAllExecutedTests().size() + getAllFailedTests().size();
- }
-
- private void parseIfRequired() {
- if (!parsed) {
- parse();
- parsed = true;
- }
- }
-
- private void parse() {
- populateTestFileList(directory);
- for(File file : testOutputFiles) {
- FileInputStream stream = null;
- try {
- stream = new FileInputStream(file);
- SAXParserFactory factory = SAXParserFactory.newInstance();
- SAXParser saxParser = factory.newSAXParser();
- saxParser.parse(new InputSource( stream ), new DefaultHandler() {
- private String name;
- private boolean failedOrErrored;
- @Override
- public void startElement(String uri, String localName, String qName, Attributes attributes) {
- if ("testcase".equals(qName)) {
- name = attributes.getValue("classname");
- failedOrErrored = false;
- if(name == null || "junit.framework.TestSuite".equals(name)) {
- name = attributes.getValue("name");
- } else {
- name = name + "." + attributes.getValue("name");
- }
- } else if (name != null) {
- if ("failure".equals(qName) || "error".equals(qName)) {
- failedOrErrored = true;
- } else if("skipped".equals(qName)) {
- name = null;
- }
- }
- }
- @Override
- public void endElement(String uri, String localName, String qName) {
- if ("testcase".equals(qName)) {
- if(name != null) {
- executedTests.add(name);
- if(failedOrErrored) {
- failedTests.add(name);
- }
- }
- }
- }
- });
- } catch (Exception e) {
- logger.error("Error parsing file " + file, e);
- } finally {
- if(stream != null) {
- try {
- stream.close();
- } catch (IOException e) {
- logger.warn("Error closing file " + file, e);
- }
- }
- }
- }
- }
-
- private void populateTestFileList(File directory) {
- File[] files = directory.listFiles();
- if(files != null) {
- for(File file : files) {
- if(file.isFile()) {
- String name = file.getName();
- if(name.startsWith("TEST-") && name.endsWith(".xml")) {
- testOutputFiles.add(file);
- int idx = name.lastIndexOf("TEST-");
- name = name.substring(idx + "TEST-".length());
- String parts[] = name.split("\\.");
- String testClass = parts[parts.length-2];
- testClassesWithAvailableReports.add(testClass);
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
deleted file mode 100644
index cf9606e..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.base.Stopwatch;
-import org.slf4j.Logger;
-
-public class LocalCommand {
-
- private static final AtomicInteger localCommandCounter = new AtomicInteger(0);
-
- private final Logger logger;
- private final Process process;
- private final StreamReader streamReader;
- private Integer exitCode;
- private final int commandId;
- private long elapsedTimeInMs;
- private final Stopwatch stopwatch = Stopwatch.createUnstarted();
-
- public LocalCommand(Logger logger, OutputPolicy outputPolicy, String command) throws IOException {
- this.commandId = localCommandCounter.incrementAndGet();
- this.logger = logger;
- logger.info("Starting LocalCommandId={}: {}", commandId, command);
- stopwatch.start();
- process = new ProcessBuilder().command(new String[] {"bash", "-c", command}).redirectErrorStream(true).start();
- streamReader = new StreamReader(outputPolicy, process.getInputStream());
- streamReader.setName("StreamReader-[" + command + "]");
- streamReader.setDaemon(true);
- streamReader.start();
- }
-
- public int getExitCode() throws InterruptedException {
- synchronized (process) {
- awaitProcessCompletion();
- return exitCode;
- }
- }
-
- public long getElapsedTimeInMs() throws InterruptedException {
- synchronized (process) {
- awaitProcessCompletion();
- return elapsedTimeInMs;
- }
- }
-
- private void awaitProcessCompletion() throws InterruptedException {
- synchronized (process) {
- if (exitCode == null) {
- exitCode = process.waitFor();
- if (stopwatch.isRunning()) {
- stopwatch.stop();
- this.elapsedTimeInMs = stopwatch.elapsed(TimeUnit.MILLISECONDS);
- logger.info("Finished LocalCommandId={}. ElapsedTime(ms)={}", commandId,
- stopwatch.elapsed(
- TimeUnit.MILLISECONDS));
- }
- }
- }
- }
-
- public void kill() {
- synchronized (process) {
- process.destroy();
- }
- }
-
- public static interface OutputPolicy {
- public void handleOutput(String line);
- public void handleThrowable(Throwable throwable);
- }
- public static class CollectLogPolicy extends CollectPolicy {
- private final Logger logger;
- public CollectLogPolicy(Logger logger) {
- this.logger = logger;
- }
- @Override
- public void handleOutput(String line) {
- logger.info(line);
- output.append(line).append("\n");
- }
- }
- public static class CollectPolicy implements OutputPolicy {
- protected final StringBuilder output = new StringBuilder();
- protected Throwable throwable;
- @Override
- public void handleOutput(String line) {
- output.append(line).append("\n");
- }
- @Override
- public void handleThrowable(Throwable throwable) {
- if(throwable instanceof IOException &&
- "Stream closed".equals(throwable.getMessage())) {
- return;
- }
- this.throwable = throwable;
- }
- public String getOutput() {
- String result = output.toString();
- if(throwable != null) {
- throw new RuntimeException(result, throwable);
- }
- return result;
- }
- }
-
- private static class StreamReader extends Thread {
- private final BufferedReader input;
- private final OutputPolicy outputPolicy;
- public StreamReader(OutputPolicy outputPolicy, InputStream in) {
- this.outputPolicy = outputPolicy;
- this.input = new BufferedReader(new InputStreamReader(in));
- }
- @Override
- public void run() {
- try {
- String line;
- while((line = input.readLine()) != null) {
- outputPolicy.handleOutput(line);
- }
- } catch(Exception e) {
- outputPolicy.handleThrowable(e);
- } finally {
- try {
- input.close();
- } catch (IOException ignored) {
-
- }
- }
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java
deleted file mode 100644
index 37f07c0..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-
-public class LocalCommandFactory {
-
- private final Logger mLogger;
-
- public LocalCommandFactory(Logger logger) {
- mLogger = logger;
- }
- public LocalCommand create(LocalCommand.CollectPolicy policy, String command)
- throws IOException {
- return new LocalCommand(mLogger, policy, command);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java
deleted file mode 100644
index a36b6f7..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class LogDirectoryCleaner extends Thread {
- private static final Logger LOG = LoggerFactory
- .getLogger(LogDirectoryCleaner.class);
- private final File mLogDir;
- private final int mMaxDirectoriesPerProfile;
- public LogDirectoryCleaner(File logDir, int maxDirectoriesPerProfile) {
- mLogDir = logDir;
- mMaxDirectoriesPerProfile = maxDirectoriesPerProfile;
- }
-
- @Override
- public void run() {
- try {
- File[] logDirs = mLogDir.listFiles();
- if(logDirs != null && logDirs.length > 0) {
- Map<String, ProfileLogs> profiles = Maps.newHashMap();
- for(File logDir : logDirs) {
- String name = logDir.getName();
- if(name.contains("-")) {
- String profile = name.substring(0, name.lastIndexOf("-"));
- ProfileLogs logs = profiles.get(profile);
- if(logs == null) {
- logs = new ProfileLogs(profile);
- profiles.put(profile, logs);
- }
- logs.dirs.add(logDir);
- }
- }
- for(String profile : profiles.keySet()) {
- ProfileLogs logs = profiles.get(profile);
- if(logs.dirs.size() > mMaxDirectoriesPerProfile) {
- File oldest = logs.getOldest();
- LOG.info("Deleting " + oldest + " from " + logs.dirs);
- FileUtils.deleteQuietly(oldest);
- }
- }
- }
- } catch(Throwable t) {
- LOG.error("Unexpected error cleaning " + mLogDir, t);
- }
- }
-
- private static class ProfileLogs {
- String name;
- List<File> dirs = Lists.newArrayList();
- ProfileLogs(String name) {
- this.name = name;
- }
- File getOldest() {
- Preconditions.checkState(!dirs.isEmpty(), "Cannot be called unless dirs.size() >= 1");
- File eldestDir = null;
- long eldestId = Long.MAX_VALUE;
- for(File dir : dirs) {
- try {
- long id = Long.parseLong(dir.getName().substring(name.length() + 1));
- if(id < eldestId) {
- eldestId = id;
- eldestDir = dir;
- }
- } catch (NumberFormatException e) {
- LOG.warn("Error parsing " + dir.getName(), e);
- }
- }
- return Preconditions.checkNotNull(eldestDir, "eldestDir");
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
deleted file mode 100644
index 4e6aa6d..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Options;
-import org.apache.hive.ptest.execution.conf.Context;
-import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
-import org.apache.hive.ptest.execution.conf.Host;
-import org.apache.hive.ptest.execution.conf.TestConfiguration;
-import org.apache.hive.ptest.execution.conf.TestParser;
-import org.apache.hive.ptest.execution.context.ExecutionContext;
-import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
-import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
-import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
-import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
-import org.apache.velocity.app.Velocity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
-import com.google.common.io.Resources;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-public class PTest {
-
- static {
- Velocity.init();
- }
- private static final Logger LOG = LoggerFactory
- .getLogger(PTest.class);
-
-
- // dummy patch
- private final TestConfiguration mConfiguration;
- private final ListeningExecutorService mExecutor;
- private final Set<String> mAddedTests;
- private final Set<String> mExecutedTests;
- private final Set<String> mFailedTests;
- private final List<Phase> mPhases;
- private final ExecutionContext mExecutionContext;
- private final Logger mLogger;
- private final List<HostExecutor> mHostExecutors;
- private final String mBuildTag;
- private final SSHCommandExecutor mSshCommandExecutor;
- private final RSyncCommandExecutor mRsyncCommandExecutor;
- private final ImmutableMap<String, String> templateDefaults;
-
- public PTest(final TestConfiguration configuration, final ExecutionContext executionContext,
- final String buildTag, final File logDir, final LocalCommandFactory localCommandFactory,
- final SSHCommandExecutor sshCommandExecutor, final RSyncCommandExecutor rsyncCommandExecutor,
- final Logger logger) throws Exception {
- mConfiguration = configuration;
- mLogger = logger;
- mBuildTag = buildTag;
- mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
- mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
- mAddedTests = new HashSet<String>();
- mExecutionContext = executionContext;
- mSshCommandExecutor = sshCommandExecutor;
- mRsyncCommandExecutor = rsyncCommandExecutor;
- mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HostExecutor %d").build()));
- final File failedLogDir = Dirs.create(new File(logDir, "failed"));
- final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
- final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
- File patchDir = Dirs.createEmpty(new File(logDir, "patches"));
- File patchFile = null;
- if(!configuration.getPatch().isEmpty()) {
- patchFile = new File(patchDir, buildTag + ".patch");
- Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile);
- }
- ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder();
- templateDefaultsBuilder.
- put("repository", configuration.getRepository()).
- put("repositoryName", configuration.getRepositoryName()).
- put("repositoryType", configuration.getRepositoryType()).
- put("buildTool", configuration.getBuildTool()).
- put("branch", configuration.getBranch()).
- put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache())).
- put("workingDir", mExecutionContext.getLocalWorkingDirectory()).
- put("buildTag", buildTag).
- put("logDir", logDir.getAbsolutePath()).
- put("javaHome", configuration.getJavaHome()).
- put("javaHomeForTests", configuration.getJavaHomeForTests()).
- put("antEnvOpts", configuration.getAntEnvOpts()).
- put("antArgs", configuration.getAntArgs()).
- put("antTestArgs", configuration.getAntTestArgs()).
- put("antTestTarget", configuration.getAntTestTarget()).
- put("mavenEnvOpts", configuration.getMavenEnvOpts()).
- put("mavenArgs", configuration.getMavenArgs()).
- put("mavenBuildArgs", configuration.getMavenBuildArgs()).
- put("mavenTestArgs", configuration.getMavenTestArgs());
- if (!Strings.isNullOrEmpty(configuration.getAdditionalProfiles())) {
- templateDefaultsBuilder.put("additionalProfiles", configuration.getAdditionalProfiles());
- }
- templateDefaults = templateDefaultsBuilder.build();
- TestParser testParser = new TestParser(configuration.getContext(), new AtomicInteger(1), configuration.getTestCasePropertyName(),
- new File(mExecutionContext.getLocalWorkingDirectory(), configuration.getRepositoryName() + "-source"),
- logger);
-
- HostExecutorBuilder hostExecutorBuilder = new HostExecutorBuilder() {
- @Override
- public HostExecutor build(Host host) {
- return new HostExecutor(host, executionContext.getPrivateKey(), mExecutor, sshCommandExecutor,
- rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10,
- configuration.shouldFetchLogsForSuccessfulTests(), logger);
- }
-
- };
- List<HostExecutor> hostExecutors = new ArrayList<HostExecutor>();
- for(Host host : mExecutionContext.getHosts()) {
- hostExecutors.add(hostExecutorBuilder.build(host));
- }
- mHostExecutors = new CopyOnWriteArrayList<HostExecutor>(hostExecutors);
- mPhases = Lists.newArrayList();
- mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults,
- configuration.getPatch(), patchFile, logger, mAddedTests));
- mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
- mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults,
- mExecutionContext.getLocalWorkingDirectory(), scratchDir, logger, logDir, patchFile));
- mPhases.add(new ExecutionPhase(mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults,
- succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger));
- mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
- }
- public int run() {
- int result = 0;
- boolean error = false;
- List<String> messages = Lists.newArrayList();
- Map<String, Long> elapsedTimes = Maps.newTreeMap();
- try {
- mLogger.info("Running tests with " + mConfiguration);
- mLogger.info("Running tests with configuration context=[{}]", mConfiguration.getContext());
- for(Phase phase : mPhases) {
- String msg = "Executing " + phase.getClass().getName();
- mLogger.info(msg);
- messages.add(msg);
- long start = System.currentTimeMillis();
- try {
- phase.execute();
- } finally {
- long elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
- TimeUnit.MILLISECONDS);
- Map<String, Long> perfMetrics = phase.getPerfMetrics();
- if (!perfMetrics.isEmpty()) {
- mLogger.info("Adding perf metrics for " + phase.getClass().getSimpleName() + " phase");
- for (Entry<String, Long> perfEntry : perfMetrics.entrySet()) {
- elapsedTimes.put(phase.getClass().getSimpleName() + "." + perfEntry.getKey(),
- TimeUnit.MINUTES.convert(perfEntry.getValue(), TimeUnit.MILLISECONDS));
- }
- }
- elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime);
- }
- }
- if(!mFailedTests.isEmpty()) {
- throw new TestsFailedException(mFailedTests.size() + " tests failed");
- }
- } catch(Throwable throwable) {
- mLogger.error("Test run exited with an unexpected error", throwable);
- // NonZeroExitCodeExceptions can have long messages and should be
- // trimmable when published to the JIRA via the JiraService
- if(throwable instanceof NonZeroExitCodeException) {
- messages.add("Tests exited with: " + throwable.getClass().getSimpleName());
- for(String line : Strings.nullToEmpty(throwable.getMessage()).split("\n")) {
- messages.add(line);
- }
- } else {
- messages.add("Tests exited with: " + throwable.getClass().getSimpleName() +
- ": " + throwable.getMessage());
- }
- error = true;
- } finally {
- for(HostExecutor hostExecutor : mHostExecutors) {
- hostExecutor.shutdownNow();
- if(hostExecutor.isBad()) {
- mExecutionContext.addBadHost(hostExecutor.getHost());
- }
- }
- mSshCommandExecutor.shutdownNow();
- mRsyncCommandExecutor.shutdownNow();
- mExecutor.shutdownNow();
- SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
- if(failedTests.isEmpty()) {
- mLogger.info(String.format("%d failed tests", failedTests.size()));
- } else {
- mLogger.warn(String.format("%d failed tests", failedTests.size()));
- }
- for(String failingTestName : failedTests) {
- mLogger.warn(failingTestName);
- }
- mLogger.info("Executed " + mExecutedTests.size() + " tests");
- for(Map.Entry<String, Long> entry : elapsedTimes.entrySet()) {
- mLogger.info(String.format("PERF: %s took %d minutes", entry.getKey(), entry.getValue()));
- }
- publishJiraComment(error, messages, failedTests, mAddedTests);
- if(error || !mFailedTests.isEmpty()) {
- result = 1;
- }
- }
- return result;
- }
-
- private void publishJiraComment(boolean error, List<String> messages, SortedSet<String> failedTests, Set<String> addedTests) {
- if(mConfiguration.getJiraName().isEmpty()) {
- mLogger.info("Skipping JIRA comment as name is empty.");
- return;
- }
- if(mConfiguration.getJiraUrl().isEmpty()) {
- mLogger.info("Skipping JIRA comment as URL is empty.");
- return;
- }
- if(mConfiguration.getJiraUser().isEmpty()) {
- mLogger.info("Skipping JIRA comment as user is empty.");
- return;
- }
- if(mConfiguration.getJiraPassword().isEmpty()) {
- mLogger.info("Skipping JIRA comment as password is empty.");
- return;
- }
- mLogger.info("Added tests: " + addedTests);
- JIRAService jira = new JIRAService(mLogger, mConfiguration, mBuildTag);
- jira.postComment(error, mExecutedTests.size(), failedTests, messages, addedTests);
- }
-
- public static class Builder {
- public PTest build(TestConfiguration configuration, ExecutionContext executionContext,
- String buildTag, File logDir, LocalCommandFactory localCommandFactory, SSHCommandExecutor sshCommandExecutor,
- RSyncCommandExecutor rsyncCommandExecutor, Logger logger) throws Exception {
- return new PTest(configuration, executionContext, buildTag, logDir, localCommandFactory, sshCommandExecutor,
- rsyncCommandExecutor, logger);
- }
- }
-
- private static final String PROPERTIES = "properties";
- private static final String SERVER_ENV_PROPERTIES = "hive.ptest.server.env.properties";
- private static final String REPOSITORY = TestConfiguration.REPOSITORY;
- private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME;
- private static final String BRANCH = TestConfiguration.BRANCH;
- private static final String PATCH = "patch";
- private static final String JAVA_HOME = TestConfiguration.JAVA_HOME;
- private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST;
- private static final String ANT_TEST_ARGS = TestConfiguration.ANT_TEST_ARGS;
- private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS;
- private static final String ANT_TEST_TARGET = TestConfiguration.ANT_TEST_TARGET;
- /**
- * All args override properties file settings except
- * for this one which is additive.
- */
- private static final String ANT_ARG = "D";
-
- public static void main(String[] args) throws Exception {
- LOG.info("Args " + Arrays.toString(args));
- CommandLineParser parser = new GnuParser();
- Options options = new Options();
- options.addOption(null, PROPERTIES, true, "properties file");
- options.addOption(null, SERVER_ENV_PROPERTIES, true, "optional properties file with environment properties");
- options.addOption(null, REPOSITORY, true, "Overrides git repository in properties file");
- options.addOption(null, REPOSITORY_NAME, true, "Overrides git repository *name* in properties file");
- options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
- options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
- options.addOption(ANT_ARG, null, true, "Supplemental ant arguments");
- options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)");
- options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
- options.addOption(null, ANT_TEST_ARGS, true, "Arguments to ant test on slave nodes only");
- options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environment variable setting");
- CommandLine commandLine = parser.parse(options, args);
- if(!commandLine.hasOption(PROPERTIES)) {
- throw new IllegalArgumentException(Joiner.on(" ").
- join(PTest.class.getName(), "--" + PROPERTIES,"config.properties"));
- }
- String testConfigurationFile = commandLine.getOptionValue(PROPERTIES);
- String environmentConfigurationFile = commandLine.getOptionValue(SERVER_ENV_PROPERTIES);
-
- Context.ContextBuilder builder = new Context.ContextBuilder();
- builder.addPropertiesFile(testConfigurationFile);
-
- if (environmentConfigurationFile != null) {
- builder.addPropertiesFile(environmentConfigurationFile);
- }
-
- Context ctx = builder.build();
-
- ExecutionContextConfiguration executionContextConfiguration = ExecutionContextConfiguration.
- withContext(ctx);
- String buildTag = System.getenv("BUILD_TAG") == null ? "undefined-"
- + System.currentTimeMillis() : System.getenv("BUILD_TAG");
- File logDir = Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag));
- LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(executionContextConfiguration.
- getGlobalLogDirectory()), 5);
- cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory());
- cleaner.setDaemon(true);
- cleaner.start();
- TestConfiguration conf = TestConfiguration.withContext(ctx, LOG);
- String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim();
- if(!repository.isEmpty()) {
- conf.setRepository(repository);
- }
- String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim();
- if(!repositoryName.isEmpty()) {
- conf.setRepositoryName(repositoryName);
- }
- String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim();
- if(!branch.isEmpty()) {
- conf.setBranch(branch);
- }
- String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim();
- if(!patch.isEmpty()) {
- conf.setPatch(patch);
- }
- String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim();
- if(!javaHome.isEmpty()) {
- conf.setJavaHome(javaHome);
- }
- String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
- if(!javaHomeForTests.isEmpty()) {
- conf.setJavaHomeForTests(javaHomeForTests);
- }
- String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim();
- if(!antTestArgs.isEmpty()) {
- conf.setAntTestArgs(antTestArgs);
- }
- String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
- if(!antEnvOpts.isEmpty()) {
- conf.setAntEnvOpts(antEnvOpts);
- }
- String antTestTarget = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_TARGET)).trim();
- if(!antTestTarget.isEmpty()) {
- conf.setAntTestTarget(antTestTarget);
- }
- String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG);
- if(supplementalAntArgs != null && supplementalAntArgs.length > 0) {
- String antArgs = Strings.nullToEmpty(conf.getAntArgs());
- if(!(antArgs.isEmpty() || antArgs.endsWith(" "))) {
- antArgs += " ";
- }
- antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs);
- conf.setAntArgs(antArgs);
- }
- ExecutionContextProvider executionContextProvider = null;
- ExecutionContext executionContext = null;
- int exitCode = 0;
- try {
- executionContextProvider = executionContextConfiguration
- .getExecutionContextProvider();
- executionContext = executionContextProvider.createExecutionContext();
- LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG);
- PTest ptest = new PTest(conf, executionContext, buildTag, logDir,
- localCommandFactory, new SSHCommandExecutor(LOG, localCommandFactory, conf.getSshOpts()),
- new RSyncCommandExecutor(LOG, 10, localCommandFactory), LOG);
- exitCode = ptest.run();
- } finally {
- if(executionContext != null) {
- executionContext.terminate();
- }
- if(executionContextProvider != null) {
- executionContextProvider.close();
- }
- }
- System.exit(exitCode);
- }
-
- @VisibleForTesting
- public Map<String, String> getTemplateDefaults() {
- return templateDefaults;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
deleted file mode 100644
index 34c66ce..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.Maps;
-import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
-import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
-import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
-import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
-import org.apache.hive.ptest.execution.ssh.SSHResult;
-import org.slf4j.Logger;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-public abstract class Phase {
-
- protected final List<HostExecutor> hostExecutors;
- private final LocalCommandFactory localCommandFactory;
- private final ImmutableMap<String, String> templateDefaults;
- protected final Logger logger;
- private Map<String, Long> perfMetrics;
-
- public Phase(List<HostExecutor> hostExecutors,
- LocalCommandFactory localCommandFactory,
- ImmutableMap<String, String> templateDefaults, Logger logger) {
- super();
- this.hostExecutors = hostExecutors;
- this.localCommandFactory = localCommandFactory;
- this.templateDefaults = templateDefaults;
- this.logger = logger;
- this.perfMetrics = new ConcurrentHashMap<>();
- }
-
- public abstract void execute() throws Throwable;
-
- // clean prep
- protected void execLocally(String command)
- throws IOException, InterruptedException, NonZeroExitCodeException {
- CollectLogPolicy localCollector = new CollectLogPolicy(logger);
- command = Templates.getTemplateResult(command, templateDefaults);
- LocalCommand localCmd = localCommandFactory.create(localCollector, command);
- if(localCmd.getExitCode() != 0) {
- throw new NonZeroExitCodeException(String.format(
- "Command '%s' failed with exit status %d and output '%s'",
- command, localCmd.getExitCode(), localCollector.getOutput()));
- }
- }
- // prep
- protected List<RemoteCommandResult> rsyncFromLocalToRemoteInstances(String localFile, String remoteFile)
- throws Exception {
- List<ListenableFuture<List<ListenableFuture<RemoteCommandResult>>>> futures = Lists.newArrayList();
- for(HostExecutor hostExecutor : hostExecutors) {
- futures.add(hostExecutor.rsyncFromLocalToRemoteInstances(localFile, remoteFile));
- }
- return flatten(futures);
- }
-
- // clean
- protected List<SSHResult> execHosts(String command)
- throws Exception {
- List<ListenableFuture<SSHResult>> futures = Lists.newArrayList();
- for(HostExecutor hostExecutor : hostExecutors) {
- futures.add(hostExecutor.exec(command));
- }
- return toListOfResults(futures);
- }
- protected List<SSHResult> execHostsIgnoreErrors(String command)
- throws Exception {
- List<ListenableFuture<SSHResult>> futures = Lists.newArrayList();
- for(HostExecutor hostExecutor : hostExecutors) {
- futures.add(hostExecutor.execIgnoreAllErrors(command));
- }
- return toListOfResults(futures, false);
- }
- // clean prep
- protected List<RemoteCommandResult> execInstances(String command)
- throws Exception {
- List<ListenableFuture<RemoteCommandResult>> futures = Lists.newArrayList();
- for(HostExecutor hostExecutor : hostExecutors) {
- futures.addAll(hostExecutor.execInstances(command));
- }
- return toListOfResults(futures);
- }
- protected List<RemoteCommandResult> initalizeHosts()
- throws Exception {
- List<ListenableFuture<List<RemoteCommandResult>>> futures = Lists.newArrayList();
- ListeningExecutorService executor = MoreExecutors.
- listeningDecorator(Executors.newFixedThreadPool(hostExecutors.size()));
- try {
- for(final HostExecutor hostExecutor : hostExecutors) {
- futures.add(executor.submit(new Callable<List<RemoteCommandResult>>() {
- @Override
- public List<RemoteCommandResult> call() throws Exception {
- return initalizeHost(hostExecutor);
- }
- }));
- }
- List<RemoteCommandResult> results = Lists.newArrayList();
- for(ListenableFuture<List<RemoteCommandResult>> future : futures) {
- List<RemoteCommandResult> result = future.get();
- if(result != null) {
- results.addAll(result);
- }
- }
- executor.shutdown();
- return results;
- } finally {
- if(executor.isShutdown()) {
- executor.shutdownNow();
- }
- }
- }
- protected List<RemoteCommandResult> initalizeHost(HostExecutor hostExecutor)
- throws Exception {
- List<RemoteCommandResult> results = Lists.newArrayList();
- results.add(hostExecutor.exec("killall -q -9 java || true").get());
- TimeUnit.SECONDS.sleep(1);
- // order matters in all of these so block
- results.addAll(toListOfResults(hostExecutor.execInstances("rm -rf $localDir/$instanceName/scratch $localDir/$instanceName/logs")));
- results.addAll(toListOfResults(hostExecutor.execInstances("mkdir -p $localDir/$instanceName/logs " +
- "$localDir/$instanceName/maven " +
- "$localDir/$instanceName/scratch " +
- "$localDir/$instanceName/ivy " +
- "$localDir/$instanceName/${repositoryName}-source")));
- // order does not matter below, so go wide
- List<ListenableFuture<List<ListenableFuture<RemoteCommandResult>>>> futures = Lists.newArrayList();
- futures.add(hostExecutor.rsyncFromLocalToRemoteInstances("$workingDir/${repositoryName}-source", "$localDir/$instanceName/"));
- futures.add(hostExecutor.rsyncFromLocalToRemoteInstances("$workingDir/maven", "$localDir/$instanceName/"));
- futures.add(hostExecutor.rsyncFromLocalToRemoteInstances("$workingDir/ivy", "$localDir/$instanceName/"));
- results.addAll(flatten(futures));
- return results;
- }
- private <T extends RemoteCommandResult> List<T> flatten(List<ListenableFuture<List<ListenableFuture<T>>>> futures)
- throws Exception {
- List<T> results = Lists.newArrayList();
- for(ListenableFuture<List<ListenableFuture<T>>> future : futures) {
- List<ListenableFuture<T>> result = future.get();
- if(result != null) {
- results.addAll(toListOfResults(result));
- }
- }
- return results;
- }
- private <T extends RemoteCommandResult> List<T> toListOfResults(List<ListenableFuture<T>> futures)
- throws Exception {
- return toListOfResults(futures, true);
- }
- private <T extends RemoteCommandResult> List<T> toListOfResults(List<ListenableFuture<T>> futures,
- boolean reportErrors)
- throws Exception {
- List<T> results = Lists.newArrayList();
- for(T result : Futures.allAsList(futures).get()) {
- if(result != null) {
- if(reportErrors && (result.getException() != null || result.getExitCode() != 0)) {
- throw new SSHExecutionException(result);
- }
- results.add(result);
- }
- }
- return results;
- }
- protected ImmutableMap<String, String> getTemplateDefaults() {
- return templateDefaults;
- }
-
- public Map<String, Long> getPerfMetrics() {
- return ImmutableMap.copyOf(perfMetrics);
- }
-
- public void addPerfMetric(final String metricKey, long value) {
- perfMetrics.put(metricKey, Long.valueOf(value));
- }
-
- public void resetPerfMetrics() {
- perfMetrics = new ConcurrentHashMap<>();
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
deleted file mode 100644
index 8fef413..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-
-import com.google.common.collect.ImmutableMap;
-
-public class PrepPhase extends Phase {
- private final File mScratchDir;
- private final File mPatchFile;
-
- public PrepPhase(List<HostExecutor> hostExecutors,
- LocalCommandFactory localCommandFactory,
- ImmutableMap<String, String> templateDefaults,
- File scratchDir, File patchFile, Logger logger) {
- super(hostExecutors, localCommandFactory, templateDefaults, logger);
- this.mScratchDir = scratchDir;
- this.mPatchFile = patchFile;
- }
- @Override
- public void execute() throws Exception {
- execLocally("rm -rf $workingDir/scratch");
- execLocally("mkdir -p $workingDir/scratch");
- if(mPatchFile != null) {
- File smartApplyPatch = new File(mScratchDir, "smart-apply-patch.sh");
- PrintWriter writer = new PrintWriter(smartApplyPatch);
- try {
- writer.write(Templates.readResource("smart-apply-patch.sh"));
- if(writer.checkError()) {
- throw new IOException("Error writing to " + smartApplyPatch);
- }
- } finally {
- writer.close();
- }
- execLocally("cp -f " + mPatchFile.getPath() + " $workingDir/scratch/build.patch");
- }
- long start;
- long elapsedTime;
- // source prep
- start = System.currentTimeMillis();
- File sourcePrepScript = new File(mScratchDir, "source-prep.sh");
- logger.info("Writing {} from template", sourcePrepScript);
- Templates.writeTemplateResult("source-prep.vm", sourcePrepScript, getTemplateDefaults());
- execLocally("bash " + sourcePrepScript.getPath());
- logger.debug("Deleting " + sourcePrepScript + ": " + sourcePrepScript.delete());
- execHostsIgnoreErrors("pkill -f java");
- execHostsIgnoreErrors("pkill -9 -f java");
- elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
- TimeUnit.MILLISECONDS);
- logger.info("PERF: source prep took " + elapsedTime + " minutes");
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ReportingPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ReportingPhase.java
deleted file mode 100644
index 12aa08b..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ReportingPhase.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.util.List;
-
-import org.slf4j.Logger;
-
-import com.google.common.collect.ImmutableMap;
-
-/**
- * Creates a tar.gz of the TEST-*.xml test results
- */
-public class ReportingPhase extends Phase {
-
- public ReportingPhase(List<HostExecutor> hostExecutors,
- LocalCommandFactory localCommandFactory,
- ImmutableMap<String, String> templateDefaults, Logger logger) {
- super(hostExecutors, localCommandFactory, templateDefaults, logger);
- }
- @Override
- public void execute() throws Exception {
- execLocally("mkdir $logDir/test-results");
- execLocally("find $logDir/{failed,succeeded} -maxdepth 2 -name 'TEST*.xml' -exec cp {} $logDir/test-results \\; 2>/dev/null");
- execLocally("cd $logDir/ && tar -zvcf test-results.tar.gz test-results/");
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Templates.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Templates.java
deleted file mode 100644
index c07869f..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Templates.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Map;
-
-import org.apache.velocity.VelocityContext;
-import org.apache.velocity.app.Velocity;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-
-public class Templates {
- public static void writeTemplateResult(String resource, File script,
- Map<String, String> keyValues) throws IOException {
- String template = readResource(resource);
- PrintWriter writer = new PrintWriter(script);
- try {
- writer.write(getTemplateResult(template, keyValues));
- if(writer.checkError()) {
- throw new IOException("Error writing to " + script);
- }
- } finally {
- writer.close();
- }
- }
- public static String readResource(String resource) throws IOException {
- return Resources.toString(Resources.getResource(resource), Charsets.UTF_8);
- }
- public static String getTemplateResult(String command, Map<String, String> keyValues)
- throws IOException {
- VelocityContext context = new VelocityContext();
- for(String key : keyValues.keySet()) {
- context.put(key, keyValues.get(key));
- }
- StringWriter writer = new StringWriter();
- if(!Velocity.evaluate(context, writer, command, command)) {
- throw new IOException("Unable to render " + command + " with " + keyValues);
- }
- writer.close();
- return writer.toString();
- }
- private Templates() {}
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java
deleted file mode 100644
index 831a909..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.collect.ImmutableMap;
-import org.slf4j.Logger;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class TestCheckPhase extends Phase {
- private final File mPatchFile;
- private final String mPatchURL;
- private Set<String> modifiedTestFiles;
- private static Cache<String, Boolean> patchUrls = CacheBuilder.newBuilder().expireAfterWrite
- (7, TimeUnit.DAYS).maximumSize(10000).build();
-
- private static final Pattern fileNameFromDiff = Pattern.compile("[/][^\\s]*");
- private static final Pattern javaTest = Pattern.compile("Test.*java");
-
- public TestCheckPhase(List<HostExecutor> hostExecutors,
- LocalCommandFactory localCommandFactory,
- ImmutableMap<String, String> templateDefaults,
- String patchUrl, File patchFile, Logger logger, Set<String> modifiedTestFiles) {
- super(hostExecutors, localCommandFactory, templateDefaults, logger);
- this.mPatchFile = patchFile;
- this.mPatchURL = patchUrl;
- this.modifiedTestFiles = modifiedTestFiles;
- }
- @Override
- public void execute() throws Exception {
- if (mPatchURL != null) {
- boolean patchUrlWasSeen = patchUrls.asMap().containsKey(mPatchURL);
- if (!patchUrlWasSeen) {
- patchUrls.put(mPatchURL, true);
- } else {
- throw new Exception("Patch URL " + mPatchURL + " was found in seen patch url's cache and " +
- "a test was probably run already on it. Aborting...");
- }
- }
- if(mPatchFile != null) {
- logger.info("Reading patchfile " + mPatchFile.getAbsolutePath());
- FileReader fr = null;
- try {
- fr = new FileReader(mPatchFile);
- BufferedReader br = new BufferedReader(fr);
- String line;
- while ((line = br.readLine()) != null) {
- if(line.startsWith("+++")) {
- logger.info("Searching line : " + line);
- Matcher fileNameMatcher = fileNameFromDiff.matcher(line);
- if (fileNameMatcher.find()) {
- String filePath = fileNameMatcher.group(0);
- String fileName = filePath.substring(filePath.lastIndexOf("/")+1);
- Matcher javaTestMatcher = javaTest.matcher(fileName);
- if (javaTestMatcher.find() || fileName.endsWith(".q")) {
- modifiedTestFiles.add(fileName);
- }
- }
- }
- }
- } finally {
- fr.close();
- }
- } else {
- logger.error("Patch file is null");
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestsFailedException.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestsFailedException.java
deleted file mode 100644
index 6889fc2..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestsFailedException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-public class TestsFailedException extends Exception {
-
- private static final long serialVersionUID = -2425197261235313197L;
- public TestsFailedException(String msg) {
- super(msg);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java
deleted file mode 100644
index 43f43f8..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hive.ptest.execution.conf.TestConfiguration;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableMap;
-
-import org.slf4j.Logger;
-
-/**
- * Wrapper phase for Yetus check execution kick-off. It will invoke ./dev-support/test-patch.sh
- * with the proper arguments and run the check itself in a asynchronous fashion.
- */
-public class YetusPhase extends Phase {
-
- private static final String YETUS_LOG_FILE = "yetus.txt";
- private static final String YETUS_OUTPUT_FOLDER = "yetus";
- private static final String YETUS_EXEC_SCRIPT = "yetus-exec.sh";
- private static final String YETUS_EXEC_VM = "yetus-exec.vm";
-
- private final File mPatchFile;
- private final File mWorkingDir;
- private final File mLogFile;
- private final File mOutputDir;
- private final File mScratchDir;
- private final String buildTag;
- private final String buildUrl;
- private final TestConfiguration conf;
-
-
- public YetusPhase(TestConfiguration configuration, List<HostExecutor> hostExecutors,
- LocalCommandFactory localCommandFactory, ImmutableMap<String, String> templateDefaults,
- String workingDir, File scratchDir, Logger logger, File logDir, File patchFile) {
-
- super(hostExecutors, localCommandFactory, templateDefaults, logger);
- this.mPatchFile = patchFile;
- this.buildTag = templateDefaults.get("buildTag");
- this.mWorkingDir = new File(workingDir, YETUS_OUTPUT_FOLDER + "_" + this.buildTag);
- this.mLogFile = new File(logDir, YETUS_LOG_FILE);
- this.mOutputDir = new File(logDir, YETUS_OUTPUT_FOLDER);
- this.mScratchDir = scratchDir;
- this.conf = configuration;
- this.buildUrl = conf.getLogsURL() + "/" + this.buildTag + "/";
-
- }
-
- /**
- * This method will start a new thread to handle the Yetus test patch script execution.
- * It creates a separate directory, instantiates the Yetus velocity template, runs it, and
- * cleans it up after.
- * @throws Exception
- */
- @Override
- public void execute() throws Exception {
-
- Thread t = new Thread(new Runnable() {
- @Override
- public void run() {
-
- if (!checkDependencies()) {
- return;
- }
- File yetusBuildScratchDir = new File(mScratchDir, buildTag);
- yetusBuildScratchDir.mkdir();
- File yetusExecScript = new File(yetusBuildScratchDir, YETUS_EXEC_SCRIPT);
- Map<String, String> templateVars = new HashMap<>();
- templateVars.putAll(getTemplateDefaults());
- templateVars.put("workingDir", mWorkingDir.getAbsolutePath());
- templateVars.put("jiraName", conf.getJiraName());
- templateVars.put("patchFile", mPatchFile.getAbsolutePath());
- templateVars.put("jiraUrl", conf.getJiraUrl());
- templateVars.put("jiraUser", conf.getJiraUser());
- templateVars.put("jiraPass", conf.getJiraPassword());
- templateVars.put("outputDir", mOutputDir.getAbsolutePath());
- templateVars.put("buildUrl", buildUrl);
- templateVars.put("buildUrlLog", YETUS_LOG_FILE);
- templateVars.put("buildUrlOutputDir", YETUS_OUTPUT_FOLDER);
- templateVars.put("logFile", mLogFile.getAbsolutePath());
- templateVars.put("yetusBuildScratchDir", yetusBuildScratchDir.getAbsolutePath());
-
- try {
- logger.info("Writing {} from template", yetusExecScript);
-
- Templates.writeTemplateResult(YETUS_EXEC_VM, yetusExecScript, templateVars);
- Process proc = new ProcessBuilder().command("bash", yetusExecScript.getPath()).start();
- int exitCode = proc.waitFor();
-
- if (exitCode == 0) {
- logger.info("Finished processing Yetus check successfully");
- }
- } catch (Exception e) {
- logger.error("Error processing Yetus check", e);
- } finally {
- logger.debug("Deleting " + yetusExecScript + ": " + yetusExecScript.delete());
- logger.debug("Deleting " + yetusBuildScratchDir + ": " + yetusBuildScratchDir.delete());
- }
- }
- });
- t.start();
- logger.info("Started Yetus check..");
- }
-
- private boolean checkDependencies(){
-
- if (mPatchFile == null || !mPatchFile.canRead()) {
- logger.error("Cannot run Yetus check - patch file is null or not readable.");
- return false;
- }
-
- if (!((mWorkingDir.isDirectory() && mWorkingDir.canWrite()) &&
- (mOutputDir.isDirectory() && mOutputDir.canWrite()))) {
- logger.error("Cannot run Yetus check - output directories not present and writable: " +
- "workingDir:%s, outputDir:%s", mWorkingDir.getAbsolutePath(), mOutputDir.getAbsolutePath());
- return false;
- }
-
- if (Strings.isNullOrEmpty(conf.getJiraUrl()) ||
- Strings.isNullOrEmpty(conf.getJiraName()) ||
- Strings.isNullOrEmpty(conf.getJiraPassword()) ||
- Strings.isNullOrEmpty(conf.getJiraUser())) {
- logger.error("Cannot run Yetus check - credentials for Jira not provided.");
- return false;
- }
-
- return true;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java
deleted file mode 100644
index f34cdba..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-
-/**
- * The context is a key-value store used to pass configuration information
- * throughout the system.
- */
-public class Context {
-
- private Map<String, String> parameters;
-
- public Context() {
- parameters = Collections.synchronizedMap(new HashMap<String, String>());
- }
- public Context(Map<String, String> paramters) {
- this();
- this.putAll(paramters);
- }
-
- /**
- * Gets a copy of the backing map structure.
- * @return immutable copy of backing map structure
- */
- public ImmutableMap<String, String> getParameters() {
- synchronized (parameters) {
- return ImmutableMap.copyOf(parameters);
- }
- }
- /**
- * Removes all of the mappings from this map.
- */
- public void clear() {
- parameters.clear();
- }
-
- /**
- * Get properties which start with a prefix. When a property is returned,
- * the prefix is removed the from name. For example, if this method is
- * called with a parameter "hdfs." and the context contains:
- * <code>
- * { hdfs.key = value, otherKey = otherValue }
- * </code>
- * this method will return a map containing:
- * <code>
- * { key = value}
- * </code>
- *
- * <b>Note:</b> The <tt>prefix</tt> must end with a period character. If not
- * this method will raise an IllegalArgumentException.
- *
- * @param prefix key prefix to find and remove from keys in resulting map
- * @return map with keys which matched prefix with prefix removed from
- * keys in resulting map. If no keys are matched, the returned map is
- * empty
- * @throws IllegalArguemntException if the given prefix does not end with
- * a period character.
- */
- public ImmutableMap<String, String> getSubProperties(String prefix) {
- Preconditions.checkArgument(prefix.endsWith("."),
- "The given prefix does not end with a period (" + prefix + ")");
- Map<String, String> result = Maps.newHashMap();
- synchronized(parameters) {
- for (String key : parameters.keySet()) {
- if (key.startsWith(prefix)) {
- String name = key.substring(prefix.length());
- result.put(name, parameters.get(key));
- }
- }
- }
- return ImmutableMap.copyOf(result);
- }
- /**
- * Associates all of the given map's keys and values in the Context.
- */
- public void putAll(Map<String, String> map) {
- parameters.putAll(map);
- }
- /**
- * Associates the specified value with the specified key in this context.
- * If the context previously contained a mapping for the key, the old value
- * is replaced by the specified value.
- * @param key key with which the specified value is to be associated
- * @param value to be associated with the specified key
- */
- public void put(String key, String value) {
- parameters.put(key, value);
- }
- /**
- * Gets value mapped to key, returning defaultValue if unmapped.
- * @param key to be found
- * @param defaultValue returned if key is unmapped
- * @return value associated with key
- */
- public Boolean getBoolean(String key, Boolean defaultValue) {
- String value = get(key);
- if(value != null) {
- return Boolean.parseBoolean(value.trim());
- }
- return defaultValue;
- }
- /**
- * Gets value mapped to key, returning null if unmapped.
- * <p>
- * Note that this method returns an object as opposed to a
- * primitive. The configuration key requested may not be mapped
- * to a value and by returning the primitive object wrapper we can
- * return null. If the key does not exist the return value of
- * this method is assigned directly to a primitive, a
- * {@link NullPointerException} will be thrown.
- * </p>
- * @param key to be found
- * @return value associated with key or null if unmapped
- */
- public Boolean getBoolean(String key) {
- return getBoolean(key, null);
- }
- /**
- * Gets value mapped to key, returning defaultValue if unmapped.
- * @param key to be found
- * @param defaultValue returned if key is unmapped
- * @return value associated with key
- */
- public Integer getInteger(String key, Integer defaultValue) {
- String value = get(key);
- if(value != null) {
- return Integer.parseInt(value.trim());
- }
- return defaultValue;
- }
- /**
- * Gets value mapped to key, returning null if unmapped.
- * <p>
- * Note that this method returns an object as opposed to a
- * primitive. The configuration key requested may not be mapped
- * to a value and by returning the primitive object wrapper we can
- * return null. If the key does not exist the return value of
- * this method is assigned directly to a primitive, a
- * {@link NullPointerException} will be thrown.
- * </p>
- * @param key to be found
- * @return value associated with key or null if unmapped
- */
- public Integer getInteger(String key) {
- return getInteger(key, null);
- }
- /**
- * Gets value mapped to key, returning defaultValue if unmapped.
- * @param key to be found
- * @param defaultValue returned if key is unmapped
- * @return value associated with key
- */
- public Long getLong(String key, Long defaultValue) {
- String value = get(key);
- if(value != null) {
- return Long.parseLong(value.trim());
- }
- return defaultValue;
- }
- /**
- * Gets value mapped to key, returning null if unmapped.
- * <p>
- * Note that this method returns an object as opposed to a
- * primitive. The configuration key requested may not be mapped
- * to a value and by returning the primitive object wrapper we can
- * return null. If the key does not exist the return value of
- * this method is assigned directly to a primitive, a
- * {@link NullPointerException} will be thrown.
- * </p>
- * @param key to be found
- * @return value associated with key or null if unmapped
- */
- public Long getLong(String key) {
- return getLong(key, null);
- }
- /**
- * Gets value mapped to key, returning defaultValue if unmapped.
- * @param key to be found
- * @param defaultValue returned if key is unmapped
- * @return value associated with key
- */
- public Float getFloat(String key, Float defaultValue) {
- String value = get(key);
- if(value != null) {
- return Float.parseFloat(value.trim());
- }
- return defaultValue;
- }
- /**
- * Gets value mapped to key, returning null if unmapped.
- * <p>
- * Note that this method returns an object as opposed to a
- * primitive. The configuration key requested may not be mapped
- * to a value and by returning the primitive object wrapper we can
- * return null. If the key does not exist the return value of
- * this method is assigned directly to a primitive, a
- * {@link NullPointerException} will be thrown.
- * </p>
- * @param key to be found
- * @return value associated with key or null if unmapped
- */
- public Float getFloat(String key) {
- return getFloat(key, null);
- }
- /**
- * Gets value mapped to key, returning defaultValue if unmapped.
- * @param key to be found
- * @param defaultValue returned if key is unmapped
- * @return value associated with key
- */
- public String getString(String key, String defaultValue) {
- return get(key, defaultValue);
- }
- /**
- * Gets value mapped to key, returning null if unmapped.
- * @param key to be found
- * @return value associated with key or null if unmapped
- */
- public String getString(String key) {
- return get(key);
- }
- private String get(String key, String defaultValue) {
- String result = parameters.get(key);
- if(result != null) {
- return result;
- }
- return defaultValue;
- }
- private String get(String key) {
- return get(key, null);
- }
- @Override
- public String toString() {
- return "{ parameters:" + parameters + " }";
- }
-
- /**
- * Build a context with the properties read from an input stream.
- * @param inputStream
- * @return context
- * @throws IOException
- */
- public static Context fromInputStream(InputStream inputStream)
- throws IOException {
- Properties properties = new Properties();
- properties.load(inputStream);
- return new Context(Maps.fromProperties(properties));
- }
-
- /**
- * Build a context with the properties read from a file
- * @param file
- * @return
- * @throws IOException
- */
- public static Context fromFile(String file) throws IOException {
- return fromFile(new File(file));
- }
-
- /**
- * Build a context with the properties read from a file
- * @param file
- * @return
- * @throws IOException
- */
- public static Context fromFile(File file) throws IOException {
- try (InputStream in = new FileInputStream(file)){
- return fromInputStream(in);
- }
- }
-
- /**
- * Builder that can aggregate properties from several files
- * when building a context. If the same key is present in more
- * than one file, the last one will be used.
- */
- public static class ContextBuilder {
-
- private Context context = new Context();
-
- /**
- * Add properties from a file to the context
- * @param file
- * @return
- * @throws IOException
- */
- public ContextBuilder addPropertiesFile(File file) throws IOException {
- try(InputStream is = new FileInputStream(file)) {
- Properties properties = new Properties();
- properties.load(is);
- context.putAll(Maps.fromProperties(properties));
- }
- return this;
- }
-
- /**
- * Add properties from a file to the context
- * @param file
- * @return
- * @throws IOException
- */
- public ContextBuilder addPropertiesFile(String file) throws IOException {
- return addPropertiesFile(new File(file));
- }
-
- /**
- * Build the context using the aggregated properties
- * @return
- * @throws IOException
- */
- public Context build() throws IOException {
- return context;
- }
- }
-}
\ No newline at end of file
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
deleted file mode 100644
index 8783e43..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hive.ptest.execution.Dirs;
-import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
-import org.apache.hive.ptest.execution.context.FixedExecutionContextProvider;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Maps;
-
-public class ExecutionContextConfiguration {
- public static final String WORKING_DIRECTORY = "workingDirectory";
- public static final String PROFILE_DIRECTORY = "profileDirectory";
- public static final String MAX_LOG_DIRS_PER_PROFILE = "maxLogDirectoriesPerProfile";
- private static final String MAX_RSYNC_THREADS = "maxRsyncThreads";
- private static final int MAX_RSYNC_THREADS_DEFAULT = 10;
- private final ExecutionContextProvider mExecutionContextProvider;
- private final String mWorkingDirectory;
- private final String mGlobalLogDirectory;
- private final String mProfileDirectory;
- private final int mMaxLogDirectoriesPerProfile;
- private final int mMaxRsyncThreads;
-
- @VisibleForTesting
- public ExecutionContextConfiguration(Context context)
- throws IOException {
- mWorkingDirectory = context.getString(WORKING_DIRECTORY, "").trim();
- Preconditions.checkArgument(!mWorkingDirectory.isEmpty(), WORKING_DIRECTORY + " is required");
- mProfileDirectory = context.getString(PROFILE_DIRECTORY, "").trim();
- Preconditions.checkArgument(!mProfileDirectory.isEmpty(), PROFILE_DIRECTORY + " is required");
- mGlobalLogDirectory = Dirs.create(new File(mWorkingDirectory, "logs")).getAbsolutePath();
- mMaxLogDirectoriesPerProfile = context.getInteger(MAX_LOG_DIRS_PER_PROFILE, 10);
- mMaxRsyncThreads = context.getInteger(MAX_RSYNC_THREADS, MAX_RSYNC_THREADS_DEFAULT);
- String executionContextProviderBuilder = context.getString("executionContextProvider",
- FixedExecutionContextProvider.Builder.class.getName()).trim();
- try {
- Object builder = Class.forName(executionContextProviderBuilder).newInstance();
- if(!(builder instanceof ExecutionContextProvider.Builder)) {
- throw new IllegalArgumentException("executionContextProvider must be of type " +
- ExecutionContextProvider.Builder.class.getName());
- }
- mExecutionContextProvider = ((ExecutionContextProvider.Builder)builder)
- .build(context, mWorkingDirectory);
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
- public int getMaxRsyncThreads() {
- return mMaxRsyncThreads;
- }
- public int getMaxLogDirectoriesPerProfile() {
- return mMaxLogDirectoriesPerProfile;
- }
- public String getWorkingDirectory() {
- return mWorkingDirectory;
- }
- public String getGlobalLogDirectory() {
- return mGlobalLogDirectory;
- }
- public String getProfileDirectory() {
- return mProfileDirectory;
- }
- public ExecutionContextProvider getExecutionContextProvider() {
- return mExecutionContextProvider;
- }
-
- public static ExecutionContextConfiguration withContext(Context ctx)
- throws IOException {
- return new ExecutionContextConfiguration(ctx);
- }
- @Override
- public String toString() {
- return "ExecutionContextConfiguration{" +
- "mExecutionContextProvider=" + mExecutionContextProvider +
- ", mWorkingDirectory='" + mWorkingDirectory + '\'' +
- ", mGlobalLogDirectory='" + mGlobalLogDirectory + '\'' +
- ", mProfileDirectory='" + mProfileDirectory + '\'' +
- ", mMaxLogDirectoriesPerProfile=" + mMaxLogDirectoriesPerProfile +
- ", mMaxRsyncThreads=" + mMaxRsyncThreads +
- '}';
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java
deleted file mode 100644
index b1eb66f..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.util.Collection;
-
-// Exists primarily to allow for easier unit tests.
-public interface FileListProvider {
-
- Collection<File> listFiles(
- File directory, String[] extensions, boolean recursive);
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
deleted file mode 100644
index a56824c..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.util.Arrays;
-
-import com.google.common.base.Preconditions;
-
-public class Host {
- private final String name;
- private final String user;
- private final int threads;
- private final String[] localDirectories;
-
- public Host(String name, String user, String[] localDirectories, Integer threads) {
- super();
- this.name = Preconditions.checkNotNull(name, "hostname");
- this.user = Preconditions.checkNotNull(user, "user");
- this.threads = Preconditions.checkNotNull(threads, "threads");
- this.localDirectories = Preconditions.checkNotNull(localDirectories, "localDirectories");
- }
- public String getName() {
- return name;
- }
- public String getUser() {
- return user;
- }
- public int getThreads() {
- return threads;
- }
- public String[] getLocalDirectories() {
- return localDirectories;
- }
- public String toShortString() {
- return name;
- }
- @Override
- public String toString() {
- return "Host [name=" + name + ", user=" + user + ", threads=" + threads
- + ", localDirectories=" + Arrays.toString(localDirectories) + "]";
- }
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(localDirectories);
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + threads;
- result = prime * result + ((user == null) ? 0 : user.hashCode());
- return result;
- }
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Host other = (Host) obj;
- if (!Arrays.equals(localDirectories, other.localDirectories))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (threads != other.threads)
- return false;
- if (user == null) {
- if (other.user != null)
- return false;
- } else if (!user.equals(other.user))
- return false;
- return true;
- }
-
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
deleted file mode 100644
index ced83bf..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-public class QFileTestBatch extends TestBatch {
-
- private final String testCasePropertyName;
- private final String driver;
- private final String queryFilesProperty;
- private final String name;
- private final String moduleName;
- private final List<String> tests;
- private final boolean isParallel;
-
- public QFileTestBatch(AtomicInteger batchIdCounter, String testCasePropertyName, String driver,
- String queryFilesProperty, Set<String> tests, boolean isParallel,
- String moduleName) {
- super(batchIdCounter);
- this.testCasePropertyName = testCasePropertyName;
- this.driver = driver;
- this.queryFilesProperty = queryFilesProperty;
- // Store as a list to have a consistent order between getTests, and the test argument generation.
- this.tests = Lists.newArrayList(tests);
- String name = Joiner.on("-").join(getBatchId(), driver, Joiner.on("-").join(
- Iterators.toArray(Iterators.limit(tests.iterator(), 3), String.class)));
- if(tests.size() > 3) {
- name = Joiner.on("-").join(name, "and", (tests.size() - 3), "more");
- }
- this.name = name;
- this.isParallel = isParallel;
- this.moduleName = moduleName;
- }
- public String getDriver() {
- return driver;
- }
- @Override
- public String getName() {
- return name;
- }
- @Override
- public String getTestArguments() {
- return String.format("-D%s=%s -D%s=%s", testCasePropertyName, driver, queryFilesProperty,
- Joiner.on(",").join(tests));
- }
-
- public Collection<String> getTests() {
- return Collections.unmodifiableList(tests);
- }
-
- @Override
- public String toString() {
- return "QFileTestBatch [batchId=" + getBatchId() + ", size=" + tests.size() + ", driver=" +
- driver + ", queryFilesProperty="
- + queryFilesProperty + ", name=" + name + ", tests=" + tests
- + ", isParallel=" + isParallel + ", moduleName=" + moduleName + "]";
- }
- @Override
- public boolean isParallel() {
- return isParallel;
- }
-
- @Override
- public String getTestModuleRelativeDir() {
- return moduleName;
- }
-
- @Override
- public int getNumTestsInBatch() {
- return tests.size();
- }
-
- @Override
- public Collection<String> getTestClasses() {
- return Collections.singleton(driver);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((driver == null) ? 0 : driver.hashCode());
- result = prime * result + (isParallel ? 1231 : 1237);
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result
- + ((queryFilesProperty == null) ? 0 : queryFilesProperty.hashCode());
- result = prime * result + ((tests == null) ? 0 : tests.hashCode());
- return result;
- }
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- QFileTestBatch other = (QFileTestBatch) obj;
- if (driver == null) {
- if (other.driver != null)
- return false;
- } else if (!driver.equals(other.driver))
- return false;
- if (isParallel != other.isParallel)
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (queryFilesProperty == null) {
- if (other.queryFilesProperty != null)
- return false;
- } else if (!queryFilesProperty.equals(other.queryFilesProperty))
- return false;
- if (tests == null) {
- if (other.tests != null)
- return false;
- } else if (!tests.equals(other.tests))
- return false;
- return true;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
deleted file mode 100644
index a83049d..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public abstract class TestBatch {
-
- public TestBatch(AtomicInteger BATCH_ID_GEN) {
- this.batchId = BATCH_ID_GEN.getAndIncrement();
- }
-
- public final int getBatchId() {
- return batchId;
- }
-
- private final int batchId;
-
- public abstract String getTestArguments();
-
- public abstract String getName();
-
- public abstract boolean isParallel();
-
- public abstract String getTestModuleRelativeDir();
-
- public abstract int getNumTestsInBatch();
-
- /* Comma separated list of classes in a batch */
- public abstract Collection<String> getTestClasses();
-
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
deleted file mode 100644
index 4e3d03c..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-
-public class TestConfiguration {
- public static final String REPOSITORY = "repository";
- public static final String REPOSITORY_NAME = "repositoryName";
- public static final String BRANCH = "branch";
- public static final String JAVA_HOME = "javaHome";
- public static final String JAVA_HOME_TEST = "javaHomeForTests";
- public static final String ANT_ENV_OPTS = "antEnvOpts";
- public static final String ANT_TEST_ARGS = "antTestArgs";
- public static final String ANT_TEST_TARGET = "antTestTarget";
-
- private static final String REPOSITORY_TYPE = "repositoryType";
- private static final String GIT = "git";
- private static final String SVN = "svn";
- private static final String ANT = "ant";
- private static final String MAVEN = "maven";
- private static final String MAVEN_ENV_OPTS = "mavenEnvOpts";
- private static final String MAVEN_ARGS = "mavenArgs";
- private static final String MAVEN_BUILD_ARGS = "mavenBuildArgs";
- private static final String MAVEN_TEST_ARGS = "mavenTestArgs";
- private static final String ADDITIONAL_PROFILES = "additionalProfiles";
- private static final String ANT_ARGS = "antArgs";
- private static final String JIRA_URL = "jiraUrl";
- private static final String JIRA_USER = "jiraUser";
- private static final String JIRA_PASSWORD = "jiraPassword";
- private static final String JENKINS_URL = "jenkinsURL";
- private static final String SSH_OPTS = "sshOpts";
- private static final String LOGS_URL = "logsURL";
- // This ends up being set to "test" | mvn ${testCasePropertyName} for instance
- private static final String TEST_CASE_PROPERTY_NAME = "testCasePropertyName";
- private static final String BUILD_TOOL = "buildTool";
- private static final String FETCH_LOGS_FOR_SUCCESSFUL_TESTS = "fetchLogsForSuccessfulTests";
- // The following parameters are not supported yet. TODO Add support
- private static final String APPLY_PATCH_SCRIPT_PATH = "applyPatchScriptPath";
- private static final String PREP_TEMPLATE_PATH = "prepTemplatePath";
- private static final String BATCH_EXEC_TEMPLATE_PATH = "batchExecTemplatePath";
-
-
- private final Context context;
- private String antArgs;
- private String antTestArgs;
- private String antEnvOpts;
- private String antTestTarget;
- private String mavenArgs;
- private String mavenBuildArgs;
- private String mavenTestArgs;
- private String mavenEnvOpts;
- private String additionalProfiles;
- private String repositoryType;
- private String repository;
- private String repositoryName;
- private String patch;
- private String javaHome;
- private String javaHomeForTests;
- private String branch;
- private String sshOpts;
- private final String jenkinsURL;
- private final String logsURL;
- private final String jiraUrl;
- private final String jiraUser;
- private final String jiraPassword;
- private final String testCasePropertyName;
- private final String buildTool;
- private final boolean fetchLogsForSuccessfulTests;
- private final String applyPathScriptPath;
- private final String prepTemplatePath;
- private final String batchExecTemplatePath;
-
- private String jiraName;
- private boolean clearLibraryCache;
-
- @VisibleForTesting
- public TestConfiguration(Context context, Logger logger)
- throws IOException {
- this.context = context;
- repositoryType = context.getString(REPOSITORY_TYPE, GIT).trim();
- repository = Preconditions.checkNotNull(context.getString(REPOSITORY), REPOSITORY).trim();
- repositoryName = Preconditions.checkNotNull(context.getString(REPOSITORY_NAME), REPOSITORY_NAME).trim();
- if(GIT.equals(repositoryType)) {
- branch = Preconditions.checkNotNull(context.getString(BRANCH), BRANCH).trim();
- } else if(SVN.equals(repositoryType)) {
- branch = Strings.nullToEmpty(null);
- } else {
- throw new IllegalArgumentException("Unkown repository type '" + repositoryType + "'");
- }
- buildTool = context.getString(BUILD_TOOL, ANT).trim();
- if(!(MAVEN.endsWith(buildTool) || ANT.equals(buildTool))) {
- throw new IllegalArgumentException("Unkown build tool type '" + buildTool + "'");
- }
- antArgs = context.getString(ANT_ARGS, "").trim();
- antTestArgs = context.getString(ANT_TEST_ARGS, "").trim();
- antEnvOpts = context.getString(ANT_ENV_OPTS, "").trim();
- antTestTarget = context.getString(ANT_TEST_TARGET, "test").trim();
- mavenArgs = context.getString(MAVEN_ARGS, "").trim();
- mavenBuildArgs = context.getString(MAVEN_BUILD_ARGS, "").trim();
- mavenTestArgs = context.getString(MAVEN_TEST_ARGS, "").trim();
- mavenEnvOpts = context.getString(MAVEN_ENV_OPTS, "").trim();
- additionalProfiles = context.getString(ADDITIONAL_PROFILES, "").trim();
- javaHome = context.getString(JAVA_HOME, "").trim();
- javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim();
- patch = Strings.nullToEmpty(null);
- jiraName = Strings.nullToEmpty(null);
- jiraUrl = context.getString(JIRA_URL, "").trim();
- jiraUser = context.getString(JIRA_USER, "").trim();
- jiraPassword = context.getString(JIRA_PASSWORD, "").trim();
- jenkinsURL = context.getString(JENKINS_URL, "https://builds.apache.org/job").trim();
- logsURL = context.getString(LOGS_URL, "").trim();
- testCasePropertyName = context.getString(TEST_CASE_PROPERTY_NAME, "testcase").trim();
- sshOpts = context.getString(SSH_OPTS, "").trim();
- fetchLogsForSuccessfulTests = context.getBoolean(FETCH_LOGS_FOR_SUCCESSFUL_TESTS, true);
-
- applyPathScriptPath = context.getString(APPLY_PATCH_SCRIPT_PATH, null);
- prepTemplatePath = context.getString(PREP_TEMPLATE_PATH, null);
- batchExecTemplatePath = context.getString(BATCH_EXEC_TEMPLATE_PATH, null);
-
- }
- public Context getContext() {
- return context;
- }
- public String getSshOpts() {
- return sshOpts;
- }
- public String getJenkinsURL() {
- return jenkinsURL;
- }
- public String getLogsURL() {
- return logsURL;
- }
- public String getJiraName() {
- return jiraName;
- }
- public void setJiraName(String jiraName) {
- this.jiraName = Strings.nullToEmpty(jiraName);
- }
- public boolean isClearLibraryCache() {
- return clearLibraryCache;
- }
- public void setClearLibraryCache(boolean clearLibraryCache) {
- this.clearLibraryCache = clearLibraryCache;
- }
- public String getBuildTool() {
- return buildTool;
- }
- public String getJiraUrl() {
- return jiraUrl;
- }
- public String getJiraUser() {
- return jiraUser;
- }
- public String getJiraPassword() {
- return jiraPassword;
- }
- public String getRepositoryType() {
- return repositoryType;
- }
- public String getRepositoryName() {
- return repositoryName;
- }
- public String getRepository() {
- return repository;
- }
- public String getBranch() {
- return branch;
- }
- public String getAntArgs() {
- return antArgs;
- }
- public String getAntTestArgs() {
- return antTestArgs;
- }
- public String getAntEnvOpts() {
- return antEnvOpts;
- }
- public String getAntTestTarget() {
- return antTestTarget;
- }
- public String getMavenArgs() {
- return mavenArgs;
- }
- public String getMavenBuildArgs() {
- return mavenBuildArgs;
- }
- public String getMavenTestArgs() { return mavenTestArgs; }
- public String getMavenEnvOpts() {
- return mavenEnvOpts;
- }
- public String getAdditionalProfiles() { return additionalProfiles; }
- public String getJavaHome() {
- return javaHome;
- }
- public String getJavaHomeForTests() {
- return javaHomeForTests;
- }
- public String getPatch() {
- return patch;
- }
- public String getTestCasePropertyName() {
- return testCasePropertyName;
- }
- public boolean shouldFetchLogsForSuccessfulTests() {return fetchLogsForSuccessfulTests;}
-
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
- public String getApplyPathScriptPath() {
- return applyPathScriptPath;
- }
-
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
- public String getPrepTemplatePath() {
- return prepTemplatePath;
- }
-
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
- public String getBatchExecTemplatePath() {
- return batchExecTemplatePath;
- }
-
- // TODO - Allow the branch to be specified as a parameter to ptest, rather than requiring a separate property file.
- // (will need to handle an alternate work-dir as well in this case - derive from branch?)
- public void setPatch(String patch) {
- this.patch = Strings.nullToEmpty(patch);
- }
- public void setRepository(String repository) {
- this.repository = Strings.nullToEmpty(repository);
- }
- public void setRepositoryName(String repositoryName) {
- this.repositoryName = Strings.nullToEmpty(repositoryName);
- }
- public void setBranch(String branch) {
- this.branch = Strings.nullToEmpty(branch);
- }
- public void setJavaHome(String javaHome) {
- this.javaHome = Strings.nullToEmpty(javaHome);
- }
- public void setJavaHomeForTests(String javaHomeForTests) {
- this.javaHomeForTests = javaHomeForTests;
- }
- public void setAntArgs(String antArgs) {
- this.antArgs = Strings.nullToEmpty(antArgs);
- }
- public void setAntTestArgs(String antTestArgs) {
- this.antTestArgs = antTestArgs;
- }
- public void setAntEnvOpts(String antEnvOpts) {
- this.antEnvOpts = Strings.nullToEmpty(antEnvOpts);
- }
- public void setAntTestTarget(String antTestTarget) {
- this.antTestTarget = Strings.nullToEmpty(antTestTarget);
- }
- public void setMavenArgs(String mavenArgs) {
- this.mavenArgs = Strings.nullToEmpty(mavenArgs);
- }
- public void setMavenTestArgs(String mavenTestArgs) {
- this.mavenTestArgs = mavenTestArgs;
- }
- public void setMavenEnvOpts(String mavenEnvOpts) {
- this.mavenEnvOpts = Strings.nullToEmpty(mavenEnvOpts);
- }
- public void setAdditionalProfiles(String additionalProfiles) { this.additionalProfiles = additionalProfiles; }
- @Override
- public String toString() {
- return "TestConfiguration [antArgs=" + antArgs + ", antTestArgs="
- + antTestArgs + ", antEnvOpts=" + antEnvOpts + ", antTestTarget="
- + antTestTarget + ", mavenArgs=" + mavenArgs + ", mavenTestArgs="
- + mavenTestArgs + ", mavenEnvOpts=" + mavenEnvOpts
- + ", repositoryType=" + repositoryType + ", repository=" + repository
- + ", repositoryName=" + repositoryName + ", patch=" + patch
- + ", javaHome=" + javaHome + ", javaHomeForTests=" + javaHomeForTests
- + ", branch=" + branch + ", jenkinsURL=" + jenkinsURL + ", jiraUrl="
- + jiraUrl + ", jiraUser=" + jiraUser + ", jiraPassword=" + jiraPassword
- + ", testCasePropertyName=" + testCasePropertyName + ", buildTool="
- + buildTool + ", jiraName=" + jiraName + ", clearLibraryCache="
- + clearLibraryCache + ", additionalProfiles=" + additionalProfiles + "]";
- }
- public static TestConfiguration withContext(Context context, Logger logger)
- throws IOException {
- return new TestConfiguration(context, logger);
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
deleted file mode 100644
index ad6dad4..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class TestParser {
-
- private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
- .trimResults().omitEmptyStrings();
-
- private static final String QTEST_MODULE_NAME = "itests/qtest";
- private static final String QTEST_SPARK_MODULE_NAME = "itests/qtest-spark";
-
- private final AtomicInteger batchIdCounter;
-
- private final Context context;
- private final String testCasePropertyName;
- private final File sourceDirectory;
- private final Logger logger;
-
- public TestParser(Context context, AtomicInteger batchIdCounter, String testCasePropertyName,
- File sourceDirectory, Logger logger) {
- this.context = context;
- this.batchIdCounter = batchIdCounter;
- this.testCasePropertyName = testCasePropertyName;
- this.sourceDirectory = sourceDirectory;
- this.logger = logger;
- }
- private List<TestBatch> parseTests() {
-
- Set<String> excluded = new HashSet<String>();
-
-
- List<TestBatch> result = Lists.newArrayList();
- for(QFileTestBatch test : parseQFileTests()) {
- result.add(test);
- excluded.add(test.getDriver());
- }
-
- Collection<TestBatch> unitTestBatches =
- new UnitTestPropertiesParser(context, batchIdCounter, testCasePropertyName, sourceDirectory, logger,
- excluded).generateTestBatches();
- result.addAll(unitTestBatches);
-
- return result;
- }
- private List<QFileTestBatch> parseQFileTests() {
- Map<String, Properties> properties = parseQTestProperties();
-
- List<QFileTestBatch> result = Lists.newArrayList();
- String qFileTestsString = context.getString("qFileTests",null);
- String []aliases;
- if (qFileTestsString != null) {
- aliases = qFileTestsString.split(" ");
- } else {
- aliases = new String[0];
- }
-
- for(String alias : aliases) {
- Context testContext = new Context(context.getSubProperties(
- Joiner.on(".").join("qFileTest", alias, "")));
- String driver = checkNotNull(testContext.getString("driver"), "driver").trim();
- // execute the driver locally?
- boolean isParallel = !testContext.getBoolean("isolateDriver", false);
- File directory = new File(sourceDirectory,
- checkNotNull(testContext.getString("directory"), "directory").trim());
- Set<String> excludedTests = Sets.newHashSet();
- for(String excludedTestGroup : TEST_SPLITTER.split(testContext.getString("exclude", ""))) {
- excludedTests.addAll(Arrays.asList(testContext.
- getString(Joiner.on(".").join("groups", excludedTestGroup), "").trim().split(" ")));
- expandTestProperties(excludedTests, properties);
- }
- Set<String> isolatedTests = Sets.newHashSet();
- for(String ioslatedTestGroup : TEST_SPLITTER.split(testContext.getString("isolate", ""))) {
- isolatedTests.addAll(Arrays.asList(testContext.
- getString(Joiner.on(".").join("groups", ioslatedTestGroup), "").trim().split(" ")));
- expandTestProperties(isolatedTests, properties);
- }
-
- Set<String> includedTests = Sets.newHashSet();
- for(String includedTestGroup : TEST_SPLITTER.split(testContext.getString("include", ""))) {
- includedTests.addAll(Arrays.asList(testContext.
- getString(Joiner.on(".").join("groups", includedTestGroup), "").trim().split(" ")));
- expandTestProperties(includedTests, properties);
- }
-
- //excluded overrides included
- includedTests.removeAll(excludedTests);
-
- result.addAll(createQFileTestBatches(
- driver,
- checkNotNull(testContext.getString("queryFilesProperty"), "queryFilesProperty").trim(),
- directory,
- testContext.getInteger("batchSize", 30),
- isParallel,
- excludedTests,
- includedTests,
- isolatedTests));
- }
- return result;
- }
-
- private List<QFileTestBatch> createQFileTestBatches(String driver, String queryFilesProperty,
- File directory, int batchSize, boolean isParallel, Set<String> excluded,
- Set<String> included, Set<String> isolated) {
- logger.info("Create batches for " + driver);
- List<String> qFileTestNames = Lists.newArrayList();
- for(File test : checkNotNull(directory.listFiles(), directory.getAbsolutePath())) {
- String testName = test.getName();
- if(test.isFile() &&
- testName.endsWith(".q") &&
- (included.isEmpty() || included.contains(testName))) {
- qFileTestNames.add(testName);
- }
- }
- List<QFileTestBatch> testBatches = Lists.newArrayList();
- List<String> testBatch = Lists.newArrayList();
- for(final String test : qFileTestNames) {
- if(excluded.contains(test)) {
- logger.info("Exlcuding test " + driver + " " + test);
- } else if(isolated.contains(test)) {
- logger.info("Executing isolated test " + driver + " " + test);
- testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
- Sets.newHashSet(test), isParallel, getModuleName(driver)));
- } else {
- if(testBatch.size() >= batchSize) {
- testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
- Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
- testBatch = Lists.newArrayList();
- }
- testBatch.add(test);
- }
- }
- if(!testBatch.isEmpty()) {
- testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
- Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
- }
- return testBatches;
- }
-
- /**
- * @return properties loaded from files specified in qFileTests.propertyFiles.${fileName}=${filePath}
- */
- private Map<String, Properties> parseQTestProperties() {
- Map<String, String> propFiles = context.getSubProperties("qFileTests.propertyFiles.");
- Map<String, Properties> propertyMap = new HashMap<String, Properties>();
- for (String propFile : propFiles.keySet()) {
- Properties properties = new Properties();
- String path = sourceDirectory + File.separator + propFiles.get(propFile);
- FileInputStream fis = null;
- try {
- fis = new FileInputStream(path);
- properties.load(fis);
- } catch (IOException e) {
- logger.warn("Error processing Qtest property file", e);
- throw new IllegalArgumentException("Error processing Qtest property file: " + path);
- } finally {
- try {
- if (fis != null) {
- fis.close();
- }
- } catch (IOException e) { //ignore
- }
- }
- propertyMap.put(propFile, properties);
- logger.info("Loaded Qtest property file: " + path);
- }
- return propertyMap;
- }
-
- /**
- * If any of given tests are of the form: ${fileName}.${property} (test list within a property file),
- * then expand them. Then remove those markers from the list of tests.
- */
- private void expandTestProperties(Set<String> tests, Map<String, Properties> propMap) {
- Set<String> toRemove = new HashSet<String>();
- Set<String> toAdd = new HashSet<String>();
-
- String pattern = "([^\\.]*)\\.\\$\\{([^}]*)}";
- Pattern r = Pattern.compile(pattern);
- for (String test : tests) {
- Matcher m = r.matcher(test);
- if (m.find()) {
- toRemove.add(test);
- logger.info("Expanding qfile property: " + test);
- String propName = m.group(1);
- String propValue = m.group(2);
- Properties props = propMap.get(propName);
- if (props == null) {
- logger.warn("No properties found for : " + propName);
- throw new IllegalArgumentException("No properties found for : " + propName);
- }
- String result = (String) props.get(propValue);
- if (result == null || result.isEmpty()) {
- logger.warn("No properties found in file: " + propName + " for property: " + propValue);
- throw new IllegalArgumentException("No propertifies found in file: " + propName + " for property: " + propValue);
- }
- Iterable<String> splits = TEST_SPLITTER.split(result);
- for (String split : splits) {
- toAdd.add(split);
- }
- }
- }
- tests.removeAll(toRemove);
- tests.addAll(toAdd);
- }
-
- private String getModuleName(String driverName) {
- if (driverName.toLowerCase().contains("spark")) {
- return QTEST_SPARK_MODULE_NAME;
- } else {
- return QTEST_MODULE_NAME;
- }
- }
-
- public Supplier<List<TestBatch>> parse() {
- return new Supplier<List<TestBatch>>() {
- @Override
- public List<TestBatch> get() {
- return parseTests();
- }
- };
- }
-
- /**
- * Manually test this against any property file.
- * @param args
- * @throws Exception
- */
- public static void main(String[] args) throws Exception {
- if (args.length < 1) {
- throw new IllegalArgumentException("Enter the property file location");
- }
- Logger log = LoggerFactory
- .getLogger(TestParser.class);
- File workingDir = new File("../..");
- File testConfigurationFile = new File(args[0]);
- final Context ctx = Context.fromFile(testConfigurationFile);
- TestConfiguration conf = TestConfiguration.withContext(ctx, log);
- TestParser testParser = new TestParser(conf.getContext(), new AtomicInteger(1), "test", workingDir, log);
- List<TestBatch> testBatches = testParser.parse().get();
- for (TestBatch testBatch : testBatches) {
- System.out.println(testBatch.getTestArguments());
- }
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
deleted file mode 100644
index 17c8abc..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hive.ptest.execution.conf;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-
-public class UnitTestBatch extends TestBatch {
-
- private final String testCasePropertyName;
- private final List<String> testList;
- private final boolean isParallel;
- private final String moduleName;
- private final String batchName;
-
- public UnitTestBatch(AtomicInteger batchIdCounter, String testCasePropertyName,
- List<String> tests, String moduleName, boolean isParallel) {
- super(batchIdCounter);
- Preconditions.checkNotNull(testCasePropertyName);
- Preconditions.checkArgument(tests!= null && !tests.isEmpty());
- this.testCasePropertyName = testCasePropertyName;
- this.testList = tests;
- this.isParallel = isParallel;
- this.moduleName = moduleName;
- if (tests.size() == 1) {
- batchName = String.format("%d_%s", getBatchId(), tests.get(0));
- } else {
- batchName = String.format("%d_UTBatch_%s_%d_tests", getBatchId(),
- (moduleName.replace("/", "__").replace(".", "__")), tests.size());
- }
- }
- @Override
- public String getTestArguments() {
- String testArg = Joiner.on(",").join(testList);
- return String.format("-D%s=%s", testCasePropertyName, testArg);
- }
-
- @Override
- public String getName() {
- // Used for logDir, failure messages etc.
- return batchName;
- }
-
- @Override
- public String toString() {
- return "UnitTestBatch [name=" + batchName + ", id=" + getBatchId() + ", moduleName=" +
- moduleName +", batchSize=" + testList.size() +
- ", isParallel=" + isParallel + ", testList=" + testList + "]";
- }
- @Override
- public boolean isParallel() {
- return isParallel;
- }
-
- @Override
- public String getTestModuleRelativeDir() {
- return moduleName;
- }
-
- @Override
- public int getNumTestsInBatch() {
- return testList.size();
- }
-
- @Override
- public Collection<String> getTestClasses() {
- return testList;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- UnitTestBatch that = (UnitTestBatch) o;
-
- if (isParallel != that.isParallel) {
- return false;
- }
- if (testList != null ? !testList.equals(that.testList) : that.testList != null) {
- return false;
- }
- if (moduleName != null ? !moduleName.equals(that.moduleName) : that.moduleName != null) {
- return false;
- }
- return batchName != null ? batchName.equals(that.batchName) : that.batchName == null;
-
- }
-
- @Override
- public int hashCode() {
- int result = testList != null ? testList.hashCode() : 0;
- result = 31 * result + (isParallel ? 1 : 0);
- result = 31 * result + (moduleName != null ? moduleName.hashCode() : 0);
- result = 31 * result + (batchName != null ? batchName.hashCode() : 0);
- return result;
- }
-}
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java
deleted file mode 100644
index d110cec..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java
+++ /dev/null
@@ -1,666 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-
-class UnitTestPropertiesParser {
-
- private static final Splitter VALUE_SPLITTER = Splitter.onPattern("[, ]")
- .trimResults().omitEmptyStrings();
-
- // Prefix for top level properties.
- static final String PROP_PREFIX_ROOT = "unitTests";
- // Prefix used to specify module specific properties. Mainly to avoid conflicts with older unitTests properties
- static final String PROP_PREFIX_MODULE = "ut";
-
- static final String PROP_DIRECTORIES = "directories";
- static final String PROP_INCLUDE = "include";
- static final String PROP_EXCLUDE = "exclude";
- static final String PROP_ISOLATE = "isolate";
- static final String PROP_SKIP_BATCHING = "skipBatching";
- static final String PROP_BATCH_SIZE = "batchSize";
- static final String PROP_SUBDIR_FOR_PREFIX = "subdirForPrefix";
-
- static final String PROP_ONE_MODULE = "module";
- static final String PROP_MODULE_LIST = "modules";
-
- private final AtomicInteger batchIdCounter;
-
- static final int DEFAULT_PROP_BATCH_SIZE = 1;
- static final int DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED = -1;
- static final int DEFAULT_PROP_BATCH_SIZE_INCLUDE_ALL = 0;
- static final String DEFAULT_PROP_DIRECTORIES = ".";
- static final String DEFAULT_PROP_SUBDIR_FOR_PREFIX = "target";
-
- static final String MODULE_NAME_TOP_LEVEL = "_root_"; // Special module for tests in the rootDir.
- static final String PREFIX_TOP_LEVEL = ".";
-
- private final Context unitRootContext; // Everything prefixed by ^unitTests.
- private final Context unitModuleContext; // Everything prefixed by ^ut.
- private final String testCasePropertyName;
- private final Logger logger;
- private final File sourceDirectory;
- private final FileListProvider fileListProvider;
- private final Set<String> excludedProvided; // excludedProvidedBy Framework vs excludedConfigured
- private final boolean inTest;
-
-
- @VisibleForTesting
- UnitTestPropertiesParser(Context testContext, AtomicInteger batchIdCounter, String testCasePropertyName,
- File sourceDirectory, Logger logger,
- FileListProvider fileListProvider,
- Set<String> excludedProvided, boolean inTest) {
- logger.info("{} created with sourceDirectory={}, testCasePropertyName={}, excludedProvide={}" +
- "fileListProvider={}, inTest={}",
- UnitTestPropertiesParser.class.getSimpleName(), sourceDirectory, testCasePropertyName,
- excludedProvided,
- (fileListProvider == null ? "null" : fileListProvider.getClass().getSimpleName()), inTest);
- Preconditions.checkNotNull(batchIdCounter, "batchIdCounter cannot be null");
- Preconditions.checkNotNull(testContext, "testContext cannot be null");
- Preconditions.checkNotNull(testCasePropertyName, "testCasePropertyName cannot be null");
- Preconditions.checkNotNull(sourceDirectory, "sourceDirectory cannot be null");
- Preconditions.checkNotNull(logger, "logger must be specified");
- this.batchIdCounter = batchIdCounter;
- this.unitRootContext =
- new Context(testContext.getSubProperties(Joiner.on(".").join(PROP_PREFIX_ROOT, "")));
- this.unitModuleContext =
- new Context(testContext.getSubProperties(Joiner.on(".").join(PROP_PREFIX_MODULE, "")));
- this.sourceDirectory = sourceDirectory;
- this.testCasePropertyName = testCasePropertyName;
- this.logger = logger;
- if (excludedProvided != null) {
- this.excludedProvided = excludedProvided;
- } else {
- this.excludedProvided = new HashSet<>();
- }
- if (fileListProvider != null) {
- this.fileListProvider = fileListProvider;
- } else {
- this.fileListProvider = new DefaultFileListProvider();
- }
- this.inTest = inTest;
-
- }
-
- UnitTestPropertiesParser(Context testContext, AtomicInteger batchIdCounter, String testCasePropertyName,
- File sourceDirectory, Logger logger,
- Set<String> excludedProvided) {
- this(testContext, batchIdCounter, testCasePropertyName, sourceDirectory, logger, null, excludedProvided, false);
- }
-
-
- Collection<TestBatch> generateTestBatches() {
- try {
- return parse();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- private Collection<TestBatch> parse() throws IOException {
-
- RootConfig rootConfig = getRootConfig(unitRootContext);
- logger.info("RootConfig: " + rootConfig);
-
- // TODO: Set this up as a tree, instead of a flat list.
- Map<String, ModuleConfig> moduleConfigs = extractModuleConfigs();
- logger.info("ModuleConfigs: {} ", moduleConfigs);
-
- List<TestDir> unitTestsDirs = processPropertyDirectories();
-
- validateConfigs(rootConfig, moduleConfigs, unitTestsDirs);
-
- LinkedHashMap<String, LinkedHashSet<TestInfo>> allTests =
- generateFullTestSet(rootConfig, moduleConfigs, unitTestsDirs);
-
-
- return createTestBatches(allTests, rootConfig, moduleConfigs);
- }
-
- private Collection<TestBatch> createTestBatches(
- LinkedHashMap<String, LinkedHashSet<TestInfo>> allTests, RootConfig rootConfig,
- Map<String, ModuleConfig> moduleConfigs) {
- List<TestBatch> testBatches = new LinkedList<>();
- for (Map.Entry<String, LinkedHashSet<TestInfo>> entry : allTests.entrySet()) {
- logger.info("Creating test batches for module={}, numTests={}", entry.getKey(),
- entry.getValue().size());
- String currentModule = entry.getKey();
- String currentPathPrefix = getPathPrefixFromModuleName(currentModule);
- int batchSize = rootConfig.batchSize;
- if (moduleConfigs.containsKey(currentModule)) {
- ModuleConfig moduleConfig = moduleConfigs.get(currentModule);
- int batchSizeModule = moduleConfig.batchSize;
- if (batchSizeModule != DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED) {
- batchSize = batchSizeModule;
- }
- }
-
- if (batchSize == DEFAULT_PROP_BATCH_SIZE_INCLUDE_ALL) {
- batchSize = Integer.MAX_VALUE;
- }
- logger.info("batchSize determined to be {} for module={}", batchSize, currentModule);
-
- // TODO Even out the batch sizes (i.e. 20/20/1 should be replaced by 14/14/13)
- List<String> currentList = new LinkedList<>();
- for (TestInfo testInfo : entry.getValue()) {
- if (testInfo.isIsolated || testInfo.skipBatching) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(batchIdCounter, testCasePropertyName, Collections.singletonList(testInfo.testName),
- currentPathPrefix, !testInfo.isIsolated);
- testBatches.add(unitTestBatch);
- } else {
- currentList.add(testInfo.testName);
- if (currentList.size() == batchSize) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(batchIdCounter, testCasePropertyName, Collections.unmodifiableList(currentList),
- currentPathPrefix, true);
- testBatches.add(unitTestBatch);
- currentList = new LinkedList<>();
- }
- }
- }
- if (!currentList.isEmpty()) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(batchIdCounter, testCasePropertyName, Collections.unmodifiableList(currentList),
- currentPathPrefix, true);
- testBatches.add(unitTestBatch);
- }
- }
- return testBatches;
- }
-
-
- private RootConfig getRootConfig(Context context) {
- ModuleConfig moduleConfig =
- getModuleConfig(context, "irrelevant", DEFAULT_PROP_BATCH_SIZE);
-
- String subDirForPrefix =
- context.getString(PROP_SUBDIR_FOR_PREFIX, DEFAULT_PROP_SUBDIR_FOR_PREFIX);
- Preconditions
- .checkArgument(StringUtils.isNotBlank(subDirForPrefix) && !subDirForPrefix.contains("/"));
-
- Context modulesContext =
- new Context(context.getSubProperties(Joiner.on(".").join(PROP_MODULE_LIST, "")));
- Set<String> includedModules = getProperty(modulesContext, PROP_INCLUDE);
- Set<String> excludedModules = getProperty(modulesContext, PROP_EXCLUDE);
- if (!includedModules.isEmpty() && !excludedModules.isEmpty()) {
- throw new IllegalArgumentException(String.format(
- "%s and %s are mutually exclusive for property %s. Provided values: included=%s, excluded=%s",
- PROP_INCLUDE, PROP_EXCLUDE, PROP_MODULE_LIST, includedModules, excludedModules));
- }
-
- return new RootConfig(includedModules, excludedModules, moduleConfig.include,
- moduleConfig.exclude, moduleConfig.skipBatching, moduleConfig.isolate,
- moduleConfig.batchSize, subDirForPrefix);
- }
-
- private ModuleConfig getModuleConfig(Context context, String moduleName, int defaultBatchSize) {
- Set<String> excluded = getProperty(context, PROP_EXCLUDE);
- Set<String> isolated = getProperty(context, PROP_ISOLATE);
- Set<String> included = getProperty(context, PROP_INCLUDE);
- Set<String> skipBatching = getProperty(context, PROP_SKIP_BATCHING);
- if (!included.isEmpty() && !excluded.isEmpty()) {
- throw new IllegalArgumentException(String.format("Included and excluded mutually exclusive." +
- " Included = %s, excluded = %s", included.toString(), excluded.toString()) +
- " for module: " + moduleName);
- }
- int batchSize = context.getInteger(PROP_BATCH_SIZE, defaultBatchSize);
-
- String pathPrefix = getPathPrefixFromModuleName(moduleName);
-
- return new ModuleConfig(moduleName, included, excluded, skipBatching, isolated, batchSize,
- pathPrefix);
- }
-
- private Set<String> getProperty(Context context, String propertyName) {
- return Sets.newHashSet(VALUE_SPLITTER.split(context.getString(propertyName, "")));
- }
-
- private String getPathPrefixFromModuleName(String moduleName) {
- String pathPrefix;
- if (moduleName.equals(MODULE_NAME_TOP_LEVEL)) {
- pathPrefix = PREFIX_TOP_LEVEL;
- } else {
- pathPrefix = moduleName.replace(".", "/");
- }
- return pathPrefix;
- }
-
- private String getModuleNameFromPathPrefix(String pathPrefix) {
- if (pathPrefix.equals(PREFIX_TOP_LEVEL)) {
- return MODULE_NAME_TOP_LEVEL;
- } else {
- pathPrefix = stripEndAndStart(pathPrefix, "/");
- pathPrefix = pathPrefix.replace("/", ".");
- // Example handling of dirs with a .
- // shims/hadoop-2.6
- // -> moduleName=shims.hadoop-.2.6
- return pathPrefix;
- }
- }
-
- private String stripEndAndStart(String srcString, String stripChars) {
- srcString = StringUtils.stripEnd(srcString, stripChars);
- srcString = StringUtils.stripStart(srcString, stripChars);
- return srcString;
- }
-
- private Map<String, ModuleConfig> extractModuleConfigs() {
- Collection<String> modules = extractConfiguredModules();
- Map<String, ModuleConfig> result = new HashMap<>();
-
- for (String moduleName : modules) {
- Context moduleContext =
- new Context(unitModuleContext.getSubProperties(Joiner.on(".").join(moduleName, "")));
- ModuleConfig moduleConfig =
- getModuleConfig(moduleContext, moduleName, DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED);
- logger.info("Adding moduleConfig={}", moduleConfig);
- result.put(moduleName, moduleConfig);
- }
- return result;
- }
-
- private Collection<String> extractConfiguredModules() {
- List<String> configuredModules = new LinkedList<>();
-
- Map<String, String> modulesMap = unitRootContext.getSubProperties(Joiner.on(".").join(
- PROP_ONE_MODULE, ""));
- for (Map.Entry<String, String> module : modulesMap.entrySet()) {
- // This is an unnecessary check, and forced configuration in the property file. Maybe
- // replace with an enforced empty value string.
- Preconditions.checkArgument(module.getKey().equals(module.getValue()));
- String moduleName = module.getKey();
- configuredModules.add(moduleName);
- }
- return configuredModules;
- }
-
- private List<TestDir> processPropertyDirectories() throws IOException {
- String srcDirString = sourceDirectory.getCanonicalPath();
- List<TestDir> unitTestsDirs = Lists.newArrayList();
- String propDirectoriies = unitRootContext.getString(PROP_DIRECTORIES, DEFAULT_PROP_DIRECTORIES);
- Iterable<String> propDirectoriesIterable = VALUE_SPLITTER.split(propDirectoriies);
-
- for (String unitTestDir : propDirectoriesIterable) {
- File unitTestParent = new File(sourceDirectory, unitTestDir);
- if (unitTestParent.isDirectory() || inTest) {
- String absUnitTestDir = unitTestParent.getCanonicalPath();
-
- Preconditions.checkState(absUnitTestDir.startsWith(srcDirString),
- "Unit test dir: " + absUnitTestDir + " is not under provided src dir: " + srcDirString);
- String modulePath = absUnitTestDir.substring(srcDirString.length());
-
- modulePath = stripEndAndStart(modulePath, "/");
-
- Preconditions.checkState(!modulePath.startsWith("/"),
- String.format("Illegal module path: [%s]", modulePath));
- if (StringUtils.isEmpty(modulePath)) {
- modulePath = PREFIX_TOP_LEVEL;
- }
- String moduleName = getModuleNameFromPathPrefix(modulePath);
- logger.info("modulePath determined as {} for testdir={}, DerivedModuleName={}", modulePath,
- absUnitTestDir, moduleName);
-
-
- logger.info("Adding unitTests dir [{}],[{}]", unitTestParent, moduleName);
- unitTestsDirs.add(new TestDir(unitTestParent, moduleName));
- } else {
- logger.warn("Unit test directory " + unitTestParent + " does not exist, or is a file.");
- }
- }
-
- return unitTestsDirs;
- }
-
- private void validateConfigs(RootConfig rootConfig,
- Map<String, ModuleConfig> moduleConfigs,
- List<TestDir> unitTestDir) {
-
- if (rootConfig.include.isEmpty() & rootConfig.exclude.isEmpty()) {
- // No conflicts. Module configuration is what will be used.
- // We've already verified that includes and excludes are not present at the same time for
- // individual modules.
- return;
- }
-
- // Validate mainly for includes / excludes working as they should.
- for (Map.Entry<String, ModuleConfig> entry : moduleConfigs.entrySet()) {
- if (rootConfig.excludedModules.contains(entry.getKey())) {
- // Don't bother validating.
- continue;
- }
-
- if (!rootConfig.includedModules.isEmpty() &&
- !rootConfig.includedModules.contains(entry.getKey())) {
- // Include specified, but this module is not in the set.
- continue;
- }
-
- // If global contains includes, individual modules can only contain additional includes.
- if (!rootConfig.include.isEmpty() && !entry.getValue().exclude.isEmpty()) {
- throw new IllegalStateException(String.format(
- "Global config specified includes, while module config for %s specified excludes",
- entry.getKey()));
... 11222 lines suppressed ...