You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/10/20 17:07:12 UTC
[hadoop] branch trunk updated: HADOOP-17767. ABFS: Update test scripts (#3124)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a996d889ec0 HADOOP-17767. ABFS: Update test scripts (#3124)
a996d889ec0 is described below
commit a996d889ec035e3b6134e94b50efc990525c3bbb
Author: Sneha Vijayarajan <sn...@gmail.com>
AuthorDate: Thu Oct 20 22:37:04 2022 +0530
HADOOP-17767. ABFS: Update test scripts (#3124)
Contributed by Sneha Vijayarajan
---
hadoop-tools/hadoop-azure/.gitignore | 3 +-
.../dev-support/testrun-scripts/runtests.sh | 166 ++++++++++--
.../dev-support/testrun-scripts/testsupport.sh | 289 ++++++++-------------
.../src/site/markdown/testing_azure.md | 115 +++++---
.../fs/azurebfs/utils/CleanupTestContainers.java | 74 ++++++
.../accountName_settings.xml.template | 185 +++++++++++++
.../test/resources/azure-auth-keys.xml.template | 156 +----------
.../hadoop-azure/src/test/resources/azure-test.xml | 45 +++-
8 files changed, 633 insertions(+), 400 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/.gitignore b/hadoop-tools/hadoop-azure/.gitignore
index 0e17efaa1eb..522210137ec 100644
--- a/hadoop-tools/hadoop-azure/.gitignore
+++ b/hadoop-tools/hadoop-azure/.gitignore
@@ -1,5 +1,6 @@
.checkstyle
bin/
-src/test/resources/combinationConfigFiles
src/test/resources/abfs-combination-test-configs.xml
dev-support/testlogs
+src/test/resources/accountSettings/*
+!src/test/resources/accountSettings/accountName_settings.xml.template
diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
index 25d9593d573..400d0a23834 100755
--- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
@@ -2,7 +2,7 @@
# shellcheck disable=SC2034
# unused variables are global in nature and used in testsupport.sh
-
+test
set -eo pipefail
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,36 +22,154 @@ set -eo pipefail
# shellcheck disable=SC1091
. dev-support/testrun-scripts/testsupport.sh
+init
+
+resourceDir=src/test/resources/
+logdir=dev-support/testlogs/
+azureTestXml=azure-auth-keys.xml
+azureTestXmlPath=$resourceDir$azureTestXml
+processCount=8
-begin
+## SECTION: TEST COMBINATION METHODS
-### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
-### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
+runHNSOAuthTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("OAuth")
+ triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
+runHNSSharedKeyTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("SharedKey")
+ triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth")
-generateconfigs
+runNonHNSSharedKeyTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("SharedKey")
+ triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=AppendBlob-HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
-generateconfigs
+runAppendBlobHNSOAuthTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+ VALUES=("OAuth" "true")
+ triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=HNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
-generateconfigs
+runTest=false
+cleanUpTestContainers=false
+echo 'Ensure below are complete before running script:'
+echo '1. Account specific settings file is present.'
+echo ' Copy accountName_settings.xml.template to accountName_settings.xml'
+echo ' where accountName in copied file name should be the test account name without domain'
+echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+echo ' folder. New account settings file to be added to same folder.)'
+echo ' Follow instructions in the template to populate settings correctly for the account'
+echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+echo ' where accountNames should be the test account names without domain'
+echo ' '
+echo ' '
+echo 'Choose action:'
+echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+do
+ case $scriptMode in
+ SET_ACTIVE_TEST_CONFIG)
+ runTest=false
+ break
+ ;;
+ RUN_TEST)
+ runTest=true
+ read -r -p "Enter parallel test run process count [default - 8]: " processCount
+ processCount=${processCount:-8}
+ break
+ ;;
+ CLEAN_UP_OLD_TEST_CONTAINERS)
+ runTest=false
+ cleanUpTestContainers=true
+ break
+ ;;
+ SET_OR_CHANGE_TEST_ACCOUNT)
+ runTest=false
+ cleanUpTestContainers=false
+ accountSettingsFile="src/test/resources/azure-auth-keys.xml"
+ if [[ ! -f "$accountSettingsFile" ]];
+ then
+ logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
+ cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
+ fi
-combination=NonHNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
-generateconfigs
+ vi $accountSettingsFile
+ exit 0
+ break
+ ;;
+ PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
+ runTest=false
+ cleanUpTestContainers=false
+ logFilePaths=/tmp/logPaths
+ find target/ -name "*output.txt" > $logFilePaths
+ logOutput "$(cat $logFilePaths)"
+ rm $logFilePaths
+ exit 0
+ break
+ ;;
+ *) logOutput "ERROR: Invalid selection"
+ ;;
+ esac
+done
+## SECTION: COMBINATION DEFINITIONS AND TRIGGER
-### DO NOT EDIT THE LINES BELOW.
+echo ' '
+echo 'Set the active test combination to run the action:'
+select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
+do
+ case $combo in
+ HNS-OAuth)
+ runHNSOAuthTest
+ break
+ ;;
+ HNS-SharedKey)
+ runHNSSharedKeyTest
+ break
+ ;;
+ nonHNS-SharedKey)
+ runNonHNSSharedKeyTest
+ break
+ ;;
+ AppendBlob-HNS-OAuth)
+ runAppendBlobHNSOAuthTest
+ break
+ ;;
+ AllCombinationsTestRun)
+ if [ $runTest == false ]
+ then
+ logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
+ break
+ fi
+ runHNSOAuthTest
+ runHNSSharedKeyTest
+ runNonHNSSharedKeyTest
+ runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
+ break
+ ;;
+ Quit)
+ exit 0
+ ;;
+ *) logOutput "ERROR: Invalid selection"
+ ;;
+ esac
+done
-runtests "$@"
+if [ $runTest == true ]
+then
+ printAggregate
+fi
diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
index 1b118ae1e82..28f96edd273 100644
--- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
@@ -15,117 +15,88 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-testresourcesdir=src/test/resources
-combconfsdir=$testresourcesdir/combinationConfigFiles
-combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
+resourceDir=src/test/resources/
+accountSettingsFolderName=accountSettings
+combtestfile=$resourceDir
+combtestfile+=abfs-combination-test-configs.xml
+logdir=dev-support/testlogs/
-logdir=dev-support/testlogs
testresultsregex="Results:(\n|.)*?Tests run:"
-testresultsfilename=
-starttime=
-threadcount=
-defaultthreadcount=8
-
-properties=
-values=
-
-validate() {
- if [ -z "$threadcount" ] ; then
- threadcount=$defaultthreadcount
- fi
- numberegex='^[0-9]+$'
- if ! [[ $threadcount =~ $numberegex ]] ; then
- echo "Exiting. The script param (threadcount) should be a number"
- exit -1
- fi
- if [ -z "$combination" ]; then
- echo "Exiting. combination cannot be empty"
- exit -1
- fi
- propertiessize=${#properties[@]}
- valuessize=${#values[@]}
- if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
- echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
- exit -1
- fi
-
- for filename in "${combinations[@]}"; do
- if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
- echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
- exit -1
- fi
- done
-}
-
-checkdependencies() {
- if ! [ "$(command -v pcregrep)" ]; then
- echo "Exiting. pcregrep is required to run the script."
- exit -1
- fi
- if ! [ "$(command -v xmlstarlet)" ]; then
- echo "Exiting. xmlstarlet is required to run the script."
- exit -1
+accountConfigFileSuffix="_settings.xml"
+testOutputLogFolder=$logdir
+testlogfilename=combinationTestLogFile
+
+fullRunStartTime=$(date +%s)
+STARTTIME=$(date +%s)
+ENDTIME=$(date +%s)
+
+outputFormatOn="\033[0;95m"
+outputFormatOff="\033[0m"
+
+triggerRun()
+{
+ echo ' '
+ combination=$1
+ accountName=$2
+ runTest=$3
+ processcount=$4
+ cleanUpTestContainers=$5
+
+ if [ -z "$accountName" ]; then
+ logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
+ exit 1;
fi
-}
-
-cleancombinationconfigs() {
- rm -rf $combconfsdir
- mkdir -p $combconfsdir
-}
-
-generateconfigs() {
- combconffile="$combconfsdir/$combination.xml"
- rm -rf "$combconffile"
- cat > "$combconffile" << ENDOFFILE
+ accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
+ rm -rf $combtestfile
+ cat > $combtestfile << ENDOFFILE
<configuration>
</configuration>
ENDOFFILE
-
- propertiessize=${#properties[@]}
- valuessize=${#values[@]}
+ propertiessize=${#PROPERTIES[@]}
+ valuessize=${#VALUES[@]}
if [ "$propertiessize" -ne "$valuessize" ]; then
- echo "Exiting. Number of properties and values differ for $combination"
- exit -1
+ logOutput "Exiting. Number of properties and values differ for $combination"
+ exit 1
fi
for ((i = 0; i < propertiessize; i++)); do
- key=${properties[$i]}
- val=${values[$i]}
+ key=${PROPERTIES[$i]}
+ val=${VALUES[$i]}
+ echo "Combination specific property setting: [ key=$key , value=$val ]"
changeconf "$key" "$val"
done
- formatxml "$combconffile"
-}
-
-formatxml() {
- xmlstarlet fo -s 2 "$1" > "$1.tmp"
- mv "$1.tmp" "$1"
-}
-
-setactiveconf() {
- if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
- echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
- exit -1
- fi
- rm -rf $combtestfile
- cat > $combtestfile << ENDOFFILE
-<configuration>
-
-</configuration>
-ENDOFFILE
+ formatxml "$combtestfile"
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
- xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
+ xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
formatxml $combtestfile
-}
+ echo ' '
+ echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
+ testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
+ touch "$testlogfilename"
-changeconf() {
- xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
- xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
- if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
+ if [ "$runTest" == true ]
then
- echo "Exiting. Changing config property failed."
- exit -1
+ STARTTIME=$(date +%s)
+ echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
+ logOutput "Test run report can be seen in $testlogfilename"
+ mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
+ ENDTIME=$(date +%s)
+ summary
fi
+
+ if [ "$cleanUpTestContainers" == true ]
+ then
+ mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
+ if grep -q "There are test failures" "$testlogfilename";
+ then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
+ pcregrep -M "$testresultsregex" "$testlogfilename"
+ exit 0
+ fi
+
+ logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
+ fi
+
}
summary() {
@@ -134,108 +105,68 @@ summary() {
echo "$combination"
echo "========================"
pcregrep -M "$testresultsregex" "$testlogfilename"
- } >> "$testresultsfilename"
+ } >> "$aggregatedTestResult"
printf "\n----- Test results -----\n"
pcregrep -M "$testresultsregex" "$testlogfilename"
-
secondstaken=$((ENDTIME - STARTTIME))
mins=$((secondstaken / 60))
secs=$((secondstaken % 60))
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
- echo "Find test logs for the combination ($combination) in: $testlogfilename"
- echo "Find consolidated test results in: $testresultsfilename"
- echo "----------"
+ echo "Find test result for the combination ($combination) in: $testlogfilename"
+ logOutput "Consolidated test result is saved in: $aggregatedTestResult"
+ echo "------------------------"
}
-init() {
- checkdependencies
- if ! mvn clean install -DskipTests
- then
- echo ""
- echo "Exiting. Build failed."
- exit -1
+checkdependencies() {
+ if ! [ "$(command -v pcregrep)" ]; then
+ logOutput "Exiting. pcregrep is required to run the script."
+ exit 1
fi
- starttime=$(date +"%Y-%m-%d_%H-%M-%S")
- mkdir -p "$logdir"
- testresultsfilename="$logdir/$starttime/Test-Results.txt"
- if [[ -z "$combinations" ]]; then
- combinations=( $( ls $combconfsdir/*.xml ))
+ if ! [ "$(command -v xmlstarlet)" ]; then
+ logOutput "Exiting. xmlstarlet is required to run the script."
+ exit 1
fi
}
-runtests() {
- parseoptions "$@"
- validate
- if [ -z "$starttime" ]; then
- init
- fi
- shopt -s nullglob
- for combconffile in "${combinations[@]}"; do
- STARTTIME=$(date +%s)
- combination=$(basename "$combconffile" .xml)
- mkdir -p "$logdir/$starttime"
- testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
- printf "\nRunning the combination: %s..." "$combination"
- setactiveconf
- mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
- ENDTIME=$(date +%s)
- summary
- done
+formatxml() {
+ xmlstarlet fo -s 2 "$1" > "$1.tmp"
+ mv "$1.tmp" "$1"
}
-begin() {
- cleancombinationconfigs
+changeconf() {
+ xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
+ xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
+ if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
+ then
+ logOutput "Exiting. Changing config property failed."
+ exit 1
+ fi
}
-parseoptions() {
-runactivate=0
-runtests=0
- while getopts ":c:a:t:" option; do
- case "${option}" in
- a)
- if [[ "$runactivate" -eq "1" ]]; then
- echo "-a Option is not multivalued"
- exit 1
- fi
- runactivate=1
- combination=$(basename "$OPTARG" .xml)
- ;;
- c)
- runtests=1
- combination=$(basename "$OPTARG" .xml)
- combinations+=("$combination")
- ;;
- t)
- threadcount=$OPTARG
- ;;
- *|?|h)
- if [[ -z "$combinations" ]]; then
- combinations=( $( ls $combconfsdir/*.xml ))
- fi
- combstr=""
- for combconffile in "${combinations[@]}"; do
- combname=$(basename "$combconffile" .xml)
- combstr="${combname}, ${combstr}"
- done
- combstr=${combstr:0:-2}
-
- echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
- echo ""
- echo "Where:"
- echo " -a COMBINATION_NAME Specify the combination name which needs to be activated."
- echo " Configured combinations: ${combstr}"
- echo " -c COMBINATION_NAME Specify the combination name for test runs"
- echo " -t THREAD_COUNT Specify the thread count"
- exit 1
- ;;
- esac
- done
- if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
- echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
+init() {
+ checkdependencies
+ if ! mvn clean install -DskipTests
+ then
+ echo ""
+ echo "Exiting. Build failed."
exit 1
fi
- if [[ "$runactivate" -eq "1" ]]; then
- setactiveconf
- exit 0
- fi
+ starttime=$(date +"%Y-%m-%d_%H-%M-%S")
+ testOutputLogFolder+=$starttime
+ mkdir -p "$testOutputLogFolder"
+ aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
+ }
+
+ printAggregate() {
+ echo :::: AGGREGATED TEST RESULT ::::
+ cat "$aggregatedTestResult"
+ fullRunEndTime=$(date +%s)
+ fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
+ mins=$((fullRunTimeInSecs / 60))
+ secs=$((fullRunTimeInSecs % 60))
+ printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
+ }
+
+logOutput() {
+ echo -e "$outputFormatOn" "$1" "$outputFormatOff"
}
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
index 933f86be3e8..e256fbef546 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -602,26 +602,76 @@ various test combinations, it will:
2. Run tests for all combinations
3. Summarize results across all the test combination runs.
-As a pre-requisite step, fill config values for test accounts and credentials
-needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
-and rename as `src/test/resources/azure-auth-keys.xml`.
+Below are the pre-requiste steps to follow:
+1. Copy
-**To add a new test combination:** Templates for mandatory test combinations
-for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
-If a new one needs to be added, add a combination set within
-`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
-and
-1. Provide a new combination name
-2. Update properties and values array which need to be effective for the test
-combination
-3. Call generateconfigs
+ ./src/test/resources/azure-auth-keys.xml.template
+ TO
+ ./src/test/resources/azure-auth-keys.xml
+ Update account names that should be used in the test run for HNS and non-HNS
+ combinations in the 2 properties present in the xml (account name should be
+ without domain part), namely
+
+ fs.azure.hnsTestAccountName
+ fs.azure.nonHnsTestAccountName
+ azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
+
+```
+XInclude is supported, so for extra security secrets may be
+kept out of the source tree then referenced through an an XInclude element:
+
+ <include xmlns="http://www.w3.org/2001/XInclude"
+ href="/users/self/.secrets/auth-keys.xml" />
+```
+
+2. Create account config files (one config file per account) in folder:
+
+ ./src/test/resources/accountSettings/
+ Follow the instruction in the start of the template file
+
+ accountName_settings.xml.template
+ within accountSettings folder while creating account config file.
+ New files created in folder accountSettings is listed in .gitignore to
+ prevent accidental cred leaks.
**To run PR validation:** Running command
-* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
-each of the combinations defined and run tests for all the combinations.
-* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
-combinations can be provided with -c option. If combinations are provided
-with -c option, tests for only those combinations will be run.
+* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
+```bash
+Choose action:
+[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
+1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT
+2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+3) CLEAN_UP_OLD_TEST_CONTAINERS
+#? 2
+```
+Enter 1: for setting active combination for IDE test run/single mvn test class runs.
+
+Enter 2: for choosing the combination to choose for mvn full test suite.
+
+Enter 3: For clean-up of any abruptly ending test leaving auto generated test
+container on the account.
+
+Enter 4: To create/modify the config file that decides the account to use for specific test combination.
+
+Enter 5: To print the log4j paths the last test runs.
+
+On next prompt, current list of combinations to choose are provided.
+Sample for Run_TEST action:
+```bash
+Enter parallel test run process count [default - 8]: 4
+Set the active test combination to run the action:
+1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun
+2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit
+#? 1
+
+Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
+
+Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
+Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
+Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
+````
+
+Provide the option for the action chosen first.
**Test logs:** Test runs will create a folder within dev-support/testlogs to
save the test logs. Folder name will be the test start timestamp. The mvn verify
@@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
Test-Results.log in the same folder. When run for PR validation, the
consolidated test results needs to be pasted into the PR comment section.
-**To generate config for use in IDE:** Running command with -a (activate) option
-`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
-the effective config relevant for the specific test combination. Hence the same
-config files used by the mvn test runs can be used for IDE without any manual
-updates needed within config file.
-
-**Other command line options:**
-* -a <COMBINATION_NAME> Specify the combination name which needs to be
-activated. This is to be used to generate config for use in IDE.
-* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
-config is specified, tests for only the specified combinations will run. All
-combinations of tests will be running if this config is not specified.
-* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
-are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
-
-In order to test ABFS, please add the following configuration to your
-`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
-compatibility tests which require WASB credentials, in addition to the ABFS
-credentials.
+**To add a new test combination:** Templates for mandatory test combinations
+for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
+If a new one needs to be added, add a combination to
+`dev-support/testrun-scripts/runtests.sh`.
+(Refer to current active combinations within
+`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
+`SECTION: TEST COMBINATION METHODS` in the script).
+
+**Test Configuration Details:**
+
+ Note that the ABFS tests include compatibility tests which require WASB
+ credentials, in addition to the ABFS credentials.
```xml
<?xml version="1.0" encoding="UTF-8"?>
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java
new file mode 100644
index 00000000000..b8272319ab8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
+
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+
+/**
+ * This looks like a test, but it is really a command to invoke to
+ * clean up containers created in other test runs.
+ *
+ */
+public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
+ private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
+ private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
+
+ public CleanupTestContainers() throws Exception {
+ }
+
+ @org.junit.Test
+ public void testDeleteContainers() throws Throwable {
+ int count = 0;
+ AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
+ String accountName = abfsConfig.getAccountName().split("\\.")[0];
+ LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
+
+ String accountKey = abfsConfig.getStorageAccountKey();
+ if ((accountKey == null) || (accountKey.isEmpty())) {
+ LOG.debug("Clean up not possible. Account ket not present in config");
+ }
+ final StorageCredentials credentials;
+ credentials = new StorageCredentialsAccountAndKey(
+ accountName, accountKey);
+ CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
+ CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+ Iterable<CloudBlobContainer> containers
+ = blobClient.listContainers(CONTAINER_PREFIX);
+ for (CloudBlobContainer container : containers) {
+ LOG.info("Container {} URI {}",
+ container.getName(),
+ container.getUri());
+ if (container.deleteIfExists()) {
+ count++;
+ LOG.info("Current deleted test containers count - #{}", count);
+ }
+ }
+ LOG.info("Summary: Deleted {} test containers", count);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template
new file mode 100644
index 00000000000..062b2f4bf3a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template
@@ -0,0 +1,185 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ NOTE:
+ Each config file will hold configs for one storage account.
+ ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
+ Below template collects information needed for both Auth settings on the account.
+
+ OAuth test mode works with token provider type - ClientCredsTokenProvider
+ For specific details on test config or pre-requisite, refer to
+ https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+
+ Pre-requisite before configuring:
+ 1. An ABFS account
+ 2. A container within account
+ 3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
+ 4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
+ 5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
+ 6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
+
+ Replace below strings with actual value:
+ ## ACCOUNT SETTINGS ##
+ 1. ACCOUNTNAME -> to account name without domain
+ 2. IS_NAMESPACE_ENABLED -> to true or false
+ 3. CONTAINER_NAME -> name of an existing container
+ 4. ACCOUNT_KEY -> Account access key
+ ## IDENTITY SETTINGS ##
+ 5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
+ 6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
+ 7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
+
+ 8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
+ 9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
+ 10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
+
+ 11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
+ 12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
+
+ 13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
+ 14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
+-->
+
+<configuration>
+
+ <!-- ACCOUNT SETTINGS -->
+ <property>
+ <name>fs.azure.abfs.account.name</name>
+ <value>ACCOUNTNAME.dfs.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.contract.test.fs.abfs</name>
+ <value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.contract.test.fs.abfss</name>
+ <value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.contract.test.fs.wasb</name>
+ <value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.wasb.account.name</name>
+ <value>ACCOUNTNAME.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.scale.test.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>IS_NAMESPACE_ENABLED</value>
+ </property>
+ <property>
+ <name>fs.azure.test.namespace.enabled</name>
+ <value>IS_NAMESPACE_ENABLED</value>
+ </property>
+ <property>
+ <name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>IS_NAMESPACE_ENABLED</value>
+ </property>
+ <property>
+ <name>fs.azure.account.hns.enabled</name>
+ <value>IS_NAMESPACE_ENABLED</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>ACCOUNT_KEY</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+ <value>ACCOUNT_KEY</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>ACCOUNT_KEY</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+ <value>ACCOUNT_KEY</value>
+ </property>
+
+ <!-- SUPERUSER SETTINGS -->
+ <property>
+ <name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.endpoint</name>
+ <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth.provider.type</name>
+ <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>SUPERUSER_CLIENT_ID</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.id</name>
+ <value>SUPERUSER_CLIENT_ID</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
+ <value>SUPERUSER_CLIENT_SECRET</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.secret</name>
+ <value>SUPERUSER_CLIENT_SECRET</value>
+ </property>
+
+ <!-- NO RBAC USER SETTINGS -->
+ <property>
+ <name>fs.azure.enable.check.access</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.azure.account.test.oauth2.client.id</name>
+ <value>NO_RBAC_USER_CLIENT_ID</value>
+ </property>
+ <property>
+ <name>fs.azure.account.test.oauth2.client.secret</name>
+ <value>NO_RBAC_USER_CLIENT_SECRET</value>
+ </property>
+ <property>
+ <name>fs.azure.check.access.testuser.guid</name>
+ <value>NO_RBAC_USER_OID</value>
+ </property>
+
+ <!-- CONTRIBUTOR RBAC USER SETTINGS -->
+ <property>
+ <name>fs.azure.account.oauth2.contributor.client.id</name>
+ <value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.contributor.client.secret</name>
+ <value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
+ </property>
+
+ <!-- READER RBAC USER SETTINGS -->
+ <property>
+ <name>fs.azure.account.oauth2.reader.client.id</name>
+ <value>READER_RBAC_USER_CLIENT_ID</value>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.reader.client.secret</name>
+ <value>READER_RBAC_USER_CLIENT_ID</value>
+ </property>
+</configuration>
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
index 12dbbfab479..636816551dd 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
@@ -14,162 +14,16 @@
-->
<configuration>
+ <!-- Account names without domain -->
- <!--====================== IMPORTANT!! ========================-->
- <!--
- 1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
- 2. UPDATE runtests.sh
- A. "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
- ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
- SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
- B. PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
- DIFFERENT VARIANTS OF CONFIGS.
- 3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
- 4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
- -->
-
- <!--=============== Auth type ===============-->
- <property>
- <name>fs.azure.account.auth.type</name>
- <value>SharedKey</value>
- </property>
-
- <!--=============== Auth related accounts ===============-->
- <!-- This set of configs needs to be provided for all the accounts with which
- the tests needs to be ran. -->
- <property>
- <name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
- <value>{ACCOUNT_ACCESS_KEY}</value>
- <description>Account access key</description>
- </property>
- <property>
- <name>
- fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
- </name>
- <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
- <description>OAuth token provider implementation class</description>
- </property>
- <property>
- <name>
- fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
- </name>
- <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
- <description>Token end point, this can be found through Azure portal
- </description>
- </property>
- <property>
- <name>
- fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
- </name>
- <value>{client id}</value>
- <description>AAD client id.</description>
- </property>
- <property>
- <name>
- fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
- </name>
- <value>{client secret}</value>
- <description>AAD client secret</description>
- </property>
-
- <!--=============== Configs for hadoop contract tests for ABFS ===============-->
- <property>
- <name>fs.contract.test.fs.abfs</name>
- <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
- </property>
- <property>
- <name>fs.contract.test.fs.abfss</name>
- <value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
- </property>
-
- <!--=============== WASB Configs ===============-->
- <property>
- <name>fs.azure.wasb.account.name</name>
- <value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
- </property>
- <property>
- <name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
- <value>WASB account key</value>
- </property>
- <property>
- <name>fs.contract.test.fs.wasb</name>
- <value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
- </value>
- </property>
-
- <!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
- <property>
- <name>fs.azure.account.oauth2.contributor.client.id</name>
- <value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
- </property>
- <property>
- <name>fs.azure.account.oauth2.contributor.client.secret</name>
- <value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
- </property>
- <property>
- <name>fs.azure.account.oauth2.reader.client.id</name>
- <value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
- </property>
- <property>
- <name>fs.azure.account.oauth2.reader.client.secret</name>
- <value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
- </property>
-
-<!--=========================== FOR CheckAccess =========================-->
-<!-- To run ABFS CheckAccess tests, you must register an app, with no role
- assignments, and set the configuration discussed below:
-
- 1) Register a new app with no RBAC
- 2) As part of the test configs you need to provide the guid for the above
-created app. Please follow the below steps to fetch the guid.
- a) Get an access token with the above created app. Please refer the
- following documentation for the same. https://docs.microsoft
-.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
- b) Decode the token fetched with the above step. You may use https
-://jwt.ms/ to decode the token
- d) The oid field in the decoded string is the guid.
- 3) Set the following configurations:
--->
-
- <property>
- <name>fs.azure.account.test.oauth2.client.id</name>
- <value>{client id}</value>
- <description>The client id(app id) for the app created on step 1
- </description>
- </property>
- <property>
- <name>fs.azure.account.test.oauth2.client.secret</name>
- <value>{client secret}</value>
- <description>
-The client secret(application's secret) for the app created on step 1
- </description>
- </property>
- <property>
- <name>fs.azure.check.access.testuser.guid</name>
- <value>{guid}</value>
- <description>The guid fetched on step 2</description>
- </property>
<property>
- <name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
-.windows.net</name>
- <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
- <description>
-Token end point. This can be found through Azure portal. As part of CheckAccess
-test cases. The access will be tested for an FS instance created with the
-above mentioned client credentials. So this configuration is necessary to
-create the test FS instance.
- </description>
+ <name>fs.azure.hnsTestAccountName</name>
+ <value></value>
</property>
- <!--========== Append blob related configs ===========-->
<property>
- <name>fs.azure.test.appendblob.enabled</name>
- <value>false</value>
- <description>If made true, tests will be running under the assumption that
- append blob is enabled and the root directory and contract test root
- directory will be part of the append blob directories. Should be false for
- non-HNS accounts.
- </description>
+ <name>fs.azure.nonHnsTestAccountName</name>
+ <value></value>
</property>
</configuration>
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
index 24ffeb5d107..6730021974e 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
@@ -28,11 +28,8 @@
<value>false</value>
</property>
- <property>
- <name>fs.azure.test.namespace.enabled</name>
- <value>true</value>
- </property>
-
+ <!--==================== ABFS CONFIGURATION ====================-->
+ <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
<property>
<name>fs.azure.abfs.latency.track</name>
<value>false</value>
@@ -43,9 +40,6 @@
<value>true</value>
</property>
- <!--==================== ABFS CONFIGURATION ====================-->
- <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
-
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
<property>
<name>fs.azure.user.agent.prefix</name>
@@ -59,7 +53,40 @@
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
- IT IN PATCHES OR COMMITS. -->
+ IT IN PATCHES OR COMMITS.
+
+ TEST SCRIPT RUNS:
+ ================
+ FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
+ TRIGGERED OVER SCRIPT:
+ ./dev-support/testrun-scripts/runtests.sh
+ (FROM hadoop-azure ROOT PROJECT PATH)
+
+ TO USE THE TEST SCRIPT,
+ 1. COPY
+ ./src/test/resources/azure-auth-keys.xml.template
+ TO
+ ./src/test/resources/azure-auth-keys.xml
+ UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
+ FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
+ PRESENT IN THE XML, NAMELY
+ fs.azure.hnsTestAccountName and
+ fs.azure.nonHnsTestAccountName
+ (ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
+
+ azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
+ ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
+
+ 2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
+ PER ACCOUNT) IN FOLDER:
+ ./src/test/resources/accountSettings/
+
+ FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
+ accountName_settings.xml.template
+ WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
+
+ NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
+ TO PREVENT ACCIDENTAL CRED LEAKS. -->
<!--=============================================================-->
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org