You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by "ASF GitHub Bot (Jira)" <ji...@apache.org> on 2022/09/20 08:39:00 UTC
[jira] [Commented] (HADOOP-17767) ABFS: Improve test scripts
[ https://issues.apache.org/jira/browse/HADOOP-17767?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17606989#comment-17606989 ]
ASF GitHub Bot commented on HADOOP-17767:
-----------------------------------------
snvijaya commented on code in PR #3124:
URL: https://github.com/apache/hadoop/pull/3124#discussion_r975054524
##########
hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh:
##########
@@ -22,36 +22,130 @@ set -eo pipefail
# shellcheck disable=SC1091
. dev-support/testrun-scripts/testsupport.sh
+init
-begin
+resourceDir=src/test/resources/
+logdir=dev-support/testlogs/
+azureTestXml=azure-auth-keys.xml
+azureTestXmlPath=$resourceDir$azureTestXml
+processCount=8
-### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
-### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
+## SECTION: TEST COMBINATION METHODS
+runHNSOAuthTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("OAuth")
+ triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth")
-generateconfigs
+runHNSSharedKeyTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("SharedKey")
+ triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=AppendBlob-HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
-generateconfigs
+runNonHNSSharedKeyTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type")
+ VALUES=("SharedKey")
+ triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=HNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
-generateconfigs
+runAppendBlobHNSOAuthTest()
+{
+ accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' $azureTestXmlPath)
+ PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+ VALUES=("OAuth" "true")
+ triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
-combination=NonHNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
-generateconfigs
+runTest=false
+cleanUpTestContainers=false
+echo 'Ensure below are complete before running script:'
+echo '1. Account specific settings file is present.'
+echo ' Copy accountName_settings.xml.template to accountName_settings.xml'
+echo ' where accountName in copied file name should be the test account name without domain'
+echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+echo ' folder. New account settings file to be added to same folder.)'
+echo ' Follow instructions in the template to populate settings correctly for the account'
+echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+echo ' where accountNames should be the test account names without domain'
+echo ' '
+echo ' '
+echo 'Choose action:'
+echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS
+do
+ case $scriptMode in
+ SET_ACTIVE_TEST_CONFIG)
+ runTest=false
+ break
+ ;;
+ RUN_TEST)
+ runTest=true
+ read -r -p "Enter parallel test run process count [default - 8]: " processCount
+ processCount=${processCount:-8}
+ break
+ ;;
+ CLEAN_UP_OLD_TEST_CONTAINERS)
+ runTest=false
+ cleanUpTestContainers=true
+ break
+ ;;
+ *) echo "ERROR: Invalid selection"
+ ;;
+ esac
+done
+## SECTION: COMBINATION DEFINITIONS AND TRIGGER
-### DO NOT EDIT THE LINES BELOW.
+echo ' '
+echo 'Set the active test combination to run the action:'
+select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth All Quit
+do
+ case $combo in
+ HNS-OAuth)
+ runHNSOAuthTest
+ break
+ ;;
+ HNS-SharedKey)
+ runHNSSharedKeyTest
+ break
+ ;;
+ nonHNS-SharedKey)
+ runNonHNSSharedKeyTest
+ break
+ ;;
+ AppendBlob-HNS-OAuth)
+ runAppendBlobHNSOAuthTest
+ break
+ ;;
+ All)
+ if [ $runTest == false ]
Review Comment:
It will print - ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG
> ABFS: Improve test scripts
> --------------------------
>
> Key: HADOOP-17767
> URL: https://issues.apache.org/jira/browse/HADOOP-17767
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.3.0
> Reporter: Sneha Vijayarajan
> Assignee: Sneha Vijayarajan
> Priority: Minor
> Labels: pull-request-available
> Fix For: 3.4.0
>
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
> Current test run scripts need manual update across all combinations in runTests.sh for account name and is working off a single azure-auth-keys.xml file. While having to test across accounts that span various geo, the config file grows big and also needs a manual change for configs such as fs.contract.test.[abfs/abfss] which has to be uniquely set. To use the script across various combinations, dev to be aware of the names of all the combinations defined in runTests.sh as well.
>
> These concerns are addressed in the new version of the scripts.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org