You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@beam.apache.org by "chamikaramj (via GitHub)" <gi...@apache.org> on 2023/03/21 18:18:58 UTC
[GitHub] [beam] chamikaramj commented on a diff in pull request #25910: Adds multi-language Python and Java quickstart release validations
chamikaramj commented on code in PR #25910:
URL: https://github.com/apache/beam/pull/25910#discussion_r1143744236
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
Review Comment:
Done.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
+ PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_input
+ PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_output
+ PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_expected_output
+ PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_sorted_output
+
+ # Cleaning up any existing input or output files
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+
+ # Generating an input file.
+ input_data=( aaa bbb ccc ddd eee)
+
+ touch $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ touch $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+
+ for item in ${input_data[*]}
+ do
+ echo $item >> $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ echo python:java:$item >> $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+ done
+
+ # Downloading the expansion service jar.
+ wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
+ JAVA_EXPANSION_SERVICE_PORT=33333
+
+ # Starting up the expansion service in a seperate shell.
+ echo "A new terminal will pop up and start a java expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
+ exec bash"
+
+ echo "Sleeping 10 seconds for the expansion service to start up."
+ sleep 10s
+
+ # Running the pipeline
+ python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
+ --runner DirectRunner \
+ --environment_type=DOCKER \
+ --input $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME \
+ --output $PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME \
+ --expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
+
+ # Validating output
+ cat ${PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME}
+
+ if cmp --silent -- $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME; then
+ echo "Successfully validated Python multi-language quickstart example."
+ else
+ { echo "Python multi-language quickstart output validation failed." ;exit 1; }
Review Comment:
Done.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
+ PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_input
+ PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_output
+ PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_expected_output
+ PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_sorted_output
+
+ # Cleaning up any existing input or output files
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+
+ # Generating an input file.
+ input_data=( aaa bbb ccc ddd eee)
+
+ touch $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ touch $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+
+ for item in ${input_data[*]}
+ do
+ echo $item >> $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ echo python:java:$item >> $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+ done
+
+ # Downloading the expansion service jar.
+ wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
+ JAVA_EXPANSION_SERVICE_PORT=33333
+
+ # Starting up the expansion service in a seperate shell.
+ echo "A new terminal will pop up and start a java expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
+ exec bash"
+
+ echo "Sleeping 10 seconds for the expansion service to start up."
+ sleep 10s
+
+ # Running the pipeline
+ python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
+ --runner DirectRunner \
+ --environment_type=DOCKER \
+ --input $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME \
+ --output $PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME \
+ --expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
+
+ # Validating output
+ cat ${PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME}
+
+ if cmp --silent -- $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME; then
+ echo "Successfully validated Python multi-language quickstart example."
+ else
+ { echo "Python multi-language quickstart output validation failed." ;exit 1; }
+ fi
+
+ # Cleaning up
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+ rm ./beam-examples-multi-language-${RELEASE_VER}.jar
Review Comment:
Moved the cleaning to the top of the test. This is to prevent corrupted output due to multiple runs. The temporary working directory should get cleaned up by the top level cleanup after the script is fully executed.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
+ PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_input
+ PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_output
+ PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_expected_output
+ PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_sorted_output
+
+ # Cleaning up any existing input or output files
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+
+ # Generating an input file.
+ input_data=( aaa bbb ccc ddd eee)
+
+ touch $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ touch $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+
+ for item in ${input_data[*]}
+ do
+ echo $item >> $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ echo python:java:$item >> $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+ done
+
+ # Downloading the expansion service jar.
+ wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
+ JAVA_EXPANSION_SERVICE_PORT=33333
+
+ # Starting up the expansion service in a seperate shell.
+ echo "A new terminal will pop up and start a java expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
+ exec bash"
+
+ echo "Sleeping 10 seconds for the expansion service to start up."
+ sleep 10s
+
+ # Running the pipeline
+ python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
+ --runner DirectRunner \
+ --environment_type=DOCKER \
+ --input $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME \
+ --output $PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME \
+ --expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
+
+ # Validating output
+ cat ${PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME}
+
+ if cmp --silent -- $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME; then
+ echo "Successfully validated Python multi-language quickstart example."
Review Comment:
Done.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
Review Comment:
Done.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
Review Comment:
I don't think there's a risk here but I think it's good to keep this as a notification for the release manager. Moved the check to the top of the script. No need for a variable since the usual pattern is to set the correct set of tests in the script.config and run the script.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
Review Comment:
Done.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
+ PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_input
+ PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_output
+ PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_expected_output
+ PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_sorted_output
+
+ # Cleaning up any existing input or output files
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+
+ # Generating an input file.
+ input_data=( aaa bbb ccc ddd eee)
+
+ touch $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ touch $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+
+ for item in ${input_data[*]}
+ do
+ echo $item >> $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ echo python:java:$item >> $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+ done
+
+ # Downloading the expansion service jar.
+ wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
+ JAVA_EXPANSION_SERVICE_PORT=33333
+
+ # Starting up the expansion service in a seperate shell.
+ echo "A new terminal will pop up and start a java expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
+ exec bash"
+
+ echo "Sleeping 10 seconds for the expansion service to start up."
+ sleep 10s
+
+ # Running the pipeline
+ python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
+ --runner DirectRunner \
+ --environment_type=DOCKER \
+ --input $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME \
+ --output $PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME \
+ --expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
+
+ # Validating output
+ cat ${PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME}
+
+ if cmp --silent -- $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME; then
+ echo "Successfully validated Python multi-language quickstart example."
+ else
+ { echo "Python multi-language quickstart output validation failed." ;exit 1; }
+ fi
+
+ # Cleaning up
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+ rm ./beam-examples-multi-language-${RELEASE_VER}.jar
+ done # Loop over Python versions.
+else
+ echo "* Skipping Python Multi-language Quickstart Validations"
+fi
+
+echo ""
+echo "====================Starting Java Multi-language Quickstart Validations==============="
+if [[ ("$java_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Java Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip[dataframe]
+
+ # Deacrivating in the main shell. We will reactivate the virtual environment new shells
+ # for the expansion service and the job server.
+ deactivate
+
+ PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT=44443
+ PYTHON_EXPANSION_SERVICE_PORT=44444
+
+ # Starting up the Job Server
+ echo "A new terminal will pop up and start a Python PortableRunner job server."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Python PortableRunner in port ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT}';
+ echo '******************************************************';
+ . ./beam_env_${py_version}/bin/activate;
+ python -m apache_beam.runners.portability.local_job_service_main -p ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT};
+ exec bash"
+
+ # Starting up the Python expansion service
+ echo "A new terminal will pop up and start a Python expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Python Portabexpansion service in port ${PYTHON_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ . ./beam_env_${py_version}/bin/activate;
+ python -m apache_beam.runners.portability.expansion_service_main --port=${PYTHON_EXPANSION_SERVICE_PORT} \
+ --fully_qualified_name_glob=* \
+ --pickle_library=cloudpickle;
+ exec bash"
+
+ echo "Sleeping 10 seconds for the job server and the expansion service to start up."
+ sleep 10s
+
+ echo '************************************************************';
+ echo '* Running Java Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ JAVA_MULTILANG_QICKSTART_FILE_PREFIX=java_multilang_quickstart
+ JAVA_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${JAVA_MULTILANG_QICKSTART_FILE_PREFIX}_output
+
+ ./gradlew :examples:multi-language:pythonDataframeWordCount -Pver=${RELEASE_VER} -Prepourl=${REPO_URL} --args=" \
+ --runner=PortableRunner \
+ --jobEndpoint=localhost:${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT} \
+ --expansionService=localhost:${PYTHON_EXPANSION_SERVICE_PORT} \
+ --output=${JAVA_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}"
+
+ # We cannot validate local output since
+ # TODO: Write output to GCS and validate when Python portable runner can forward credentials to GCS appropriately.
+
+ java_xlang_quickstart_status=$?
+ if [[ $java_xlang_quickstart_status -eq 0 ]]; then
+ echo "Successfully completed Java multi-language quickstart example."
+ else
+ { echo "Java multi-language quickstart failed." ;exit 1; }
Review Comment:
Unfortunately we cannot validate output with DirectRunner for Java pipelines yet but left a TODO to add this in the future. The script will fail if the pipeline failed though. I think this is adequate as a validation for the expansion service.
##########
release/src/main/scripts/run_rc_validation.sh:
##########
@@ -484,8 +484,240 @@ else
echo "* Skipping Python Leaderboard & GameStates Validations"
fi
+# Setting up Docker images for multi-language quickstart tests.
+if [[ ("$python_xlang_quickstart" = true) \
+ || ("$java_xlang_quickstart" = true) ]]; then
+ echo "[Confirmation Required] Multi-language quickstart tests requie generating
+ final Docker tags for several candidate Docker images. This should be safe to do
+ so since these tags will be overridden during the Docker image finalization step
+ of the Beam release.
+ Continue with the release validation ? [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Cannot continue with the validation. Exiting."
+ exit
+ fi
+
+ RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
+ FINAL_DOCKER_TAG=${RELEASE_VER}
+
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
+ PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
+ docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
+ done
+
+ JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
+ JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
+ docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
+ echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
+ docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
+fi
+
+echo ""
+echo "====================Starting Python Multi-language Quickstart Validations==============="
+if [[ ("$python_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip
+
+ echo '************************************************************';
+ echo '* Running Python Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ PYTHON_MULTILANG_QICKSTART_FILE_PREFIX=python_multilang_quickstart
+ PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_input
+ PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_output
+ PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_expected_output
+ PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}_sorted_output
+
+ # Cleaning up any existing input or output files
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+
+ # Generating an input file.
+ input_data=( aaa bbb ccc ddd eee)
+
+ touch $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ touch $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+
+ for item in ${input_data[*]}
+ do
+ echo $item >> $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME
+ echo python:java:$item >> $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME
+ done
+
+ # Downloading the expansion service jar.
+ wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
+ JAVA_EXPANSION_SERVICE_PORT=33333
+
+ # Starting up the expansion service in a seperate shell.
+ echo "A new terminal will pop up and start a java expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
+ exec bash"
+
+ echo "Sleeping 10 seconds for the expansion service to start up."
+ sleep 10s
+
+ # Running the pipeline
+ python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
+ --runner DirectRunner \
+ --environment_type=DOCKER \
+ --input $PYTHON_MULTILANG_QICKSTART_INPUT_FILE_NAME \
+ --output $PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME \
+ --expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
+
+ # Validating output
+ cat ${PYTHON_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME}
+
+ if cmp --silent -- $PYTHON_MULTILANG_QICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QICKSTART_SORTED_OUTPUT_FILE_NAME; then
+ echo "Successfully validated Python multi-language quickstart example."
+ else
+ { echo "Python multi-language quickstart output validation failed." ;exit 1; }
+ fi
+
+ # Cleaning up
+ rm ${PYTHON_MULTILANG_QICKSTART_FILE_PREFIX}*
+ rm ./beam-examples-multi-language-${RELEASE_VER}.jar
+ done # Loop over Python versions.
+else
+ echo "* Skipping Python Multi-language Quickstart Validations"
+fi
+
+echo ""
+echo "====================Starting Java Multi-language Quickstart Validations==============="
+if [[ ("$java_xlang_quickstart" = true) \
+ && ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
+ cd ${LOCAL_BEAM_DIR}
+
+ echo "---------------------Downloading Python Staging RC----------------------------"
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ { echo "Failed to download Python Staging RC files." ;exit 1; }
+ fi
+
+ echo "--------------------------Verifying Hashes------------------------------------"
+ sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+
+ `which pip` install --upgrade pip
+ `which pip` install --upgrade setuptools
+
+ echo "-----------------------Setting up Shell Env Vars------------------------------"
+ set_bashrc
+
+ # Run Java Multi-language pipelines under multiple versions of Python using DirectRunner
+ cd ${LOCAL_BEAM_DIR}
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version interpreter----------------"
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
+ ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
+
+ echo "--------------------------Installing Python SDK-------------------------------"
+ pip install apache-beam-${RELEASE_VER}.zip[dataframe]
+
+ # Deacrivating in the main shell. We will reactivate the virtual environment new shells
+ # for the expansion service and the job server.
+ deactivate
+
+ PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT=44443
+ PYTHON_EXPANSION_SERVICE_PORT=44444
+
+ # Starting up the Job Server
+ echo "A new terminal will pop up and start a Python PortableRunner job server."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Python PortableRunner in port ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT}';
+ echo '******************************************************';
+ . ./beam_env_${py_version}/bin/activate;
+ python -m apache_beam.runners.portability.local_job_service_main -p ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT};
+ exec bash"
+
+ # Starting up the Python expansion service
+ echo "A new terminal will pop up and start a Python expansion service."
+ gnome-terminal -x sh -c \
+ "echo '******************************************************';
+ echo '* Running Python Portabexpansion service in port ${PYTHON_EXPANSION_SERVICE_PORT}';
+ echo '******************************************************';
+ . ./beam_env_${py_version}/bin/activate;
+ python -m apache_beam.runners.portability.expansion_service_main --port=${PYTHON_EXPANSION_SERVICE_PORT} \
+ --fully_qualified_name_glob=* \
+ --pickle_library=cloudpickle;
+ exec bash"
+
+ echo "Sleeping 10 seconds for the job server and the expansion service to start up."
+ sleep 10s
+
+ echo '************************************************************';
+ echo '* Running Java Multi-language Quickstart with DirectRunner';
+ echo '************************************************************';
+
+ JAVA_MULTILANG_QICKSTART_FILE_PREFIX=java_multilang_quickstart
+ JAVA_MULTILANG_QICKSTART_OUTPUT_FILE_NAME=${JAVA_MULTILANG_QICKSTART_FILE_PREFIX}_output
+
+ ./gradlew :examples:multi-language:pythonDataframeWordCount -Pver=${RELEASE_VER} -Prepourl=${REPO_URL} --args=" \
+ --runner=PortableRunner \
+ --jobEndpoint=localhost:${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT} \
+ --expansionService=localhost:${PYTHON_EXPANSION_SERVICE_PORT} \
+ --output=${JAVA_MULTILANG_QICKSTART_OUTPUT_FILE_NAME}"
+
+ # We cannot validate local output since
+ # TODO: Write output to GCS and validate when Python portable runner can forward credentials to GCS appropriately.
+
+ java_xlang_quickstart_status=$?
+ if [[ $java_xlang_quickstart_status -eq 0 ]]; then
+ echo "Successfully completed Java multi-language quickstart example."
+ else
+ { echo "Java multi-language quickstart failed." ;exit 1; }
Review Comment:
Updated comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org