You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ch...@apache.org on 2024/03/04 19:01:07 UTC
(beam) 01/01: Debugging Transform Service test suite.
This is an automated email from the ASF dual-hosted git repository.
chamikara pushed a commit to branch transform_service_test_suite
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 27bb78b2ae5c9f7d5e0cea569a78335850454a00
Author: Chamikara Jayalath <ch...@gmail.com>
AuthorDate: Mon Mar 4 10:59:15 2024 -0800
Debugging Transform Service test suite.
---
.../beam_PostCommit_TransformService_Direct.yml | 8 +++---
sdks/python/expansion-service-container/boot.go | 14 +++++++++-
sdks/python/scripts/run_transform_service.sh | 32 ++++++++++++++++++++++
3 files changed, 49 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
index 1497681ae99..5efca5b481d 100644
--- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml
+++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
@@ -62,7 +62,7 @@ jobs:
matrix:
job_name: ["beam_PostCommit_TransformService_Direct"]
job_phrase: ["Run TransformService_Direct PostCommit"]
- python_version: ['3.8','3.11']
+ python_version: ['3.11']
steps:
- uses: actions/checkout@v4
- name: Setup repository
@@ -75,9 +75,7 @@ jobs:
uses: ./.github/actions/setup-environment-action
with:
java-version: 11
- python-version: |
- 3.8
- 3.11
+ python-version: 3.11
- name: run TransformService Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -87,6 +85,8 @@ jobs:
-Pjava11Home=$JAVA_HOME_11_X64 \
-PuseWheelDistribution \
-PpythonVersion=${{ matrix.python_version }} \
+ --stacktrace \
+ --info \
- name: Archive Python Test Results
uses: actions/upload-artifact@v4
if: failure()
diff --git a/sdks/python/expansion-service-container/boot.go b/sdks/python/expansion-service-container/boot.go
index ba56b349c4e..3c6636fd846 100644
--- a/sdks/python/expansion-service-container/boot.go
+++ b/sdks/python/expansion-service-container/boot.go
@@ -24,6 +24,7 @@ import (
"io/ioutil"
"log"
"os"
+// "os/exec"
"path/filepath"
"strconv"
"strings"
@@ -137,7 +138,7 @@ func launchExpansionServiceProcess() error {
if err != nil {
return err
}
- log.Printf("Starting Python expansion service ...")
+ log.Printf("Starting Python expansion service ... 1")
dir := filepath.Join("/opt/apache/beam", venvDirectory)
os.Setenv("VIRTUAL_ENV", dir)
@@ -145,6 +146,17 @@ func launchExpansionServiceProcess() error {
args := []string{"-m", expansionServiceEntrypoint, "-p", strconv.Itoa(*port), "--fully_qualified_name_glob", "*"}
+
+ log.Printf("****** xyz123 executing ls -al:")
+ if err := execx.Execute("/usr/bin/ls", "-al"); err != nil {
+ return fmt.Errorf("Could not execute /usr/bin/ls -al: %s", err)
+ }
+
+ log.Printf("****** xyz123 executing ls -al /:")
+ if err := execx.Execute("/usr/bin/ls", "-al", "/"); err != nil {
+ return fmt.Errorf("Could not execute /usr/bin/ls -al /: %s", err)
+ }
+
if *requirements_file != "" {
log.Printf("Received the requirements file %v", *requirements_file)
updatedRequirementsFileName, err := getUpdatedRequirementsFile(*requirements_file, *dependencies_dir)
diff --git a/sdks/python/scripts/run_transform_service.sh b/sdks/python/scripts/run_transform_service.sh
index b71d67707e0..b1596112555 100755
--- a/sdks/python/scripts/run_transform_service.sh
+++ b/sdks/python/scripts/run_transform_service.sh
@@ -16,6 +16,8 @@
# limitations under the License.
#
+echo "************ xyz123 run_transform_service.sh: 1"
+
read -r -d '' USAGE <<END
Usage: run_expansion_services.sh (start|stop) [options]
Options:
@@ -26,6 +28,8 @@ Options:
--stop [command to stop the transform service for the given group_id]
END
+echo "************ xyz123 run_transform_service.sh: 2"
+
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
@@ -66,6 +70,8 @@ while [[ $# -gt 0 ]]; do
esac
done
+echo "************ xyz123 run_transform_service.sh: 3"
+
FILE_BASE="beam-transform-service"
if [ -n "$GROUP_ID" ]; then
FILE_BASE="$FILE_BASE-$GROUP_ID"
@@ -73,12 +79,35 @@ fi
TEMP_DIR=/tmp
+echo "************ xyz123 run_transform_service.sh: 4"
+
case $STARTSTOP in
start)
+ echo "************ xyz123 in script run_transform_service.sh: starting the service"
echo "Starting the transform service for project $GROUP_ID at port $EXTERNAL_PORT for Beam version $BEAM_VERSION_DOCKER transform service startup jar is $TRANSFORM_SERVICE_LAUNCHER_JAR"
java -jar $TRANSFORM_SERVICE_LAUNCHER_JAR --project_name $GROUP_ID --port $EXTERNAL_PORT --beam_version $BEAM_VERSION_DOCKER --command up >$TEMP_DIR/$FILE_BASE-java1.log 2>&1 </dev/null
+ echo "************ xyz123 in script run_transform_service.sh: DONE starting the service"
;;
stop)
+ echo "************ xyz123 run_transform_service.sh: printing docker logs"
+ temp_output=`docker ps`
+ printf "temp_output: $temp_output"
+ printf "Logs from Controller:\n"
+ temp_output=`docker ps | grep 'controller'`
+ printf "temp_output controller: $temp_output"
+ container=${temp_output%% *}
+ docker logs $container
+ printf "Logs from Java expansion service:\n"
+ temp_output=`docker ps | grep 'java'`
+ printf "temp_output java exp service: $temp_output"
+ container=${temp_output%% *}
+ docker logs $container
+ printf "Logs from Python expansion service:\n"
+ temp_output=`docker ps | grep 'python'`
+ printf "temp_output py exp service: $temp_output"
+ container=${temp_output%% *}
+ docker logs $container
+ echo "************ xyz123 run_transform_service.sh: DONE printing docker logs"
echo "Stopping the transform service for project $GROUP_ID at port $EXTERNAL_PORT for Beam version $BEAM_VERSION_DOCKER transform service startup jar is $TRANSFORM_SERVICE_LAUNCHER_JAR"
java -jar $TRANSFORM_SERVICE_LAUNCHER_JAR --project_name $GROUP_ID --port $EXTERNAL_PORT --beam_version $BEAM_VERSION_DOCKER --command down >$TEMP_DIR/$FILE_BASE-java2.log 2>&1 </dev/null
TRANSFORM_SERVICE_TEMP_DIR=$TEMP_DIR/$GROUP_ID
@@ -88,3 +117,6 @@ case $STARTSTOP in
fi
;;
esac
+
+echo "************ xyz123 run_transform_service.sh: 5"
+