You are viewing a plain text version of this content. The canonical link for it is here.

Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/04/02 00:12:32 UTC

Build failed in Jenkins: beam_PostCommit_Go_VR_Flink #4571

See <https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/4571/display/redirect?page=changes>

Changes:

[rohde.samuel] Update Dataflow V1Beta3 to newest version to add the

[rohde.samuel] Add the 'enable_hot_key_logging' PipelineOption and plumb it to the

[lpost] [BEAM-12079] Deterministic coding enforcement causes

[noreply] fix: variables names in test-stream.md

[Udi Meiri] Update dev Dataflow containers to latest version.

[Kyle Weaver] [BEAM-12033] Create ZetaSqlException, which contains a GRPC Status code.

[noreply] Merge pull request #14372 from [BEAM-12014] Add BIGNUMERIC support in


------------------------------------------
[...truncated 76.57 KB...]
        ;;
    --expansion_addr)
        EXPANSION_ADDR="$2"
        shift # past argument
        shift # past value
        ;;
    --jenkins)
        JENKINS=true
        shift # past argument
        ;;
    *)    # unknown option
        echo "Unknown option: $1"
        exit 1
        ;;
esac
done

# Go to the root of the repository
cd $(git rev-parse --show-toplevel)
git rev-parse --show-toplevel

# Verify in the root of the repository
test -d sdks/go/test

# Hacky python script to find a free port. Note there is a small chance the chosen port could
# get taken before being claimed by the job server.
SOCKET_SCRIPT="
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', 0))
print(s.getsockname()[1])
s.close()
"

# Set up environment based on runner.
if [[ "$RUNNER" == "dataflow" ]]; then
  if [[ -z "$DATAFLOW_WORKER_JAR" ]]; then
    DATAFLOW_WORKER_JAR=$(find $(pwd)/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-*.jar)
  fi
  echo "Using Dataflow worker jar: $DATAFLOW_WORKER_JAR"

  if [[ -z "$EXPANSION_ADDR" && -n "$EXPANSION_SERVICE_JAR" ]]; then
    EXPANSION_PORT=$(python -c "$SOCKET_SCRIPT")
    EXPANSION_ADDR="localhost:$EXPANSION_PORT"
    echo "No expansion address specified; starting a new expansion server on $EXPANSION_ADDR"
    java -jar $EXPANSION_SERVICE_JAR $EXPANSION_PORT &
    EXPANSION_PID=$!
  fi
elif [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "portable" ]]; then
  if [[ -z "$ENDPOINT" ]]; then
    JOB_PORT=$(python -c "$SOCKET_SCRIPT")
    ENDPOINT="localhost:$JOB_PORT"
    echo "No endpoint specified; starting a new $RUNNER job server on $ENDPOINT"
    if [[ "$RUNNER" == "flink" ]]; then
      java \
          -jar $FLINK_JOB_SERVER_JAR \
          --flink-master [local] \
          --job-port $JOB_PORT \
          --expansion-port 0 \
          --artifact-port 0 &
    elif [[ "$RUNNER" == "spark" ]]; then
      java \
          -jar $SPARK_JOB_SERVER_JAR \
          --spark-master-url local \
          --job-port $JOB_PORT \
          --expansion-port 0 \
          --artifact-port 0 &
      ARGS="-p 1" # Spark runner fails if jobs are run concurrently.
    elif [[ "$RUNNER" == "portable" ]]; then
      python \
          -m apache_beam.runners.portability.local_job_service_main \
          --port $JOB_PORT &
    else
      echo "Unknown runner: $RUNNER"
      exit 1;
    fi
    JOBSERVER_PID=$!
  fi

  if [[ -z "$EXPANSION_ADDR" && -n "$EXPANSION_SERVICE_JAR" ]]; then
    EXPANSION_PORT=$(python -c "$SOCKET_SCRIPT")
    EXPANSION_ADDR="localhost:$EXPANSION_PORT"
    echo "No expansion address specified; starting a new expansion server on $EXPANSION_ADDR"
    java -jar $EXPANSION_SERVICE_JAR $EXPANSION_PORT &
    EXPANSION_PID=$!
  fi
fi
python -c "$SOCKET_SCRIPT"
python -c "$SOCKET_SCRIPT"

if [[ "$RUNNER" == "dataflow" ]]; then
  # Verify docker and gcloud commands exist
  command -v docker
  docker -v
  command -v gcloud
  gcloud --version

  # ensure gcloud is version 186 or above
  TMPDIR=$(mktemp -d)
  gcloud_ver=$(gcloud -v | head -1 | awk '{print $4}')
  if [[ "$gcloud_ver" < "186" ]]
  then
    pushd $TMPDIR
    curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-186.0.0-linux-x86_64.tar.gz --output gcloud.tar.gz
    tar xf gcloud.tar.gz
    ./google-cloud-sdk/install.sh --quiet
    . ./google-cloud-sdk/path.bash.inc
    popd
    gcloud components update --quiet || echo 'gcloud components update failed'
    gcloud -v
  fi

  # Build the container
  TAG=$(date +%Y%m%d-%H%M%S)
  CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk
  echo "Using container $CONTAINER"
  ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG

  # Verify it exists
  docker images | grep $TAG

  # Push the container
  gcloud docker -- push $CONTAINER
else
  TAG=dev
  ./gradlew :sdks:go:container:docker -Pdocker-tag=$TAG
  CONTAINER=apache/beam_go_sdk
fi
Apr 02, 2021 12:11:33 AM org.apache.beam.sdk.expansion.service.ExpansionService loadRegisteredTransforms
INFO: Registering external transforms: [beam:transforms:xlang:test:cgbk, beam:transforms:xlang:test:flatten, beam:transforms:xlang:test:prefix, beam:transforms:xlang:test:multi, beam:transforms:xlang:test:gbk, beam:transforms:xlang:test:comgl, beam:transforms:xlang:test:compk, beam:transforms:xlang:count, beam:transforms:xlang:filter_less_than_eq, beam:transforms:xlang:test:partition, beam:transforms:xlang:test:parquet_write, beam:transforms:xlang:parquet_read, beam:transforms:xlang:textio_read, beam:external:java:generate_sequence:v1]
Apr 02, 2021 12:11:33 AM org.apache.beam.runners.jobsubmission.JobServerDriver createArtifactStagingService
INFO: ArtifactStagingService started on localhost:32849
Apr 02, 2021 12:11:33 AM org.apache.beam.runners.jobsubmission.JobServerDriver createExpansionService
INFO: Java ExpansionService started on localhost:46239
Apr 02, 2021 12:11:33 AM org.apache.beam.runners.jobsubmission.JobServerDriver createJobServer
INFO: JobService started on localhost:35981
Apr 02, 2021 12:11:33 AM org.apache.beam.runners.jobsubmission.JobServerDriver run
INFO: Job server now running, terminate with Ctrl+C

ARGS="$ARGS --timeout=$TIMEOUT"
ARGS="$ARGS --runner=$RUNNER"
ARGS="$ARGS --project=$DATAFLOW_PROJECT"
ARGS="$ARGS --region=$REGION"
ARGS="$ARGS --environment_type=DOCKER"
ARGS="$ARGS --environment_config=$CONTAINER:$TAG"
ARGS="$ARGS --staging_location=$GCS_LOCATION/staging-validatesrunner-test"
ARGS="$ARGS --temp_location=$GCS_LOCATION/temp-validatesrunner-test"
ARGS="$ARGS --dataflow_worker_jar=$DATAFLOW_WORKER_JAR"
ARGS="$ARGS --endpoint=$ENDPOINT"
if [[ -n "$EXPANSION_ADDR" ]]; then
  ARGS="$ARGS --expansion_addr=$EXPANSION_ADDR"
fi

# Running "go test" requires some additional setup on Jenkins.
if [[ "$JENKINS" == true ]]; then
  # Copy the go repo as it is on Jenkins, to ensure we compile with the code
  # being tested.
  cd ..
  mkdir -p temp_gopath/src/github.com/apache/beam/sdks
  cp -a ./src/sdks/go ./temp_gopath/src/github.com/apache/beam/sdks
  TEMP_GOPATH=$(pwd)/temp_gopath
  cd ./src

  echo ">>> RUNNING $RUNNER VALIDATESRUNNER TESTS"
  GOPATH=$TEMP_GOPATH go test -v github.com/apache/beam/sdks/go/test/integration/... $ARGS \
      || TEST_EXIT_CODE=$? # don't fail fast here; clean up environment before exiting
else
  echo ">>> RUNNING $RUNNER VALIDATESRUNNER TESTS"
  go test -v ./sdks/go/test/integration/... $ARGS \
      || TEST_EXIT_CODE=$? # don't fail fast here; clean up environment before exiting
fi
sdks/go/test/integration/integration.go:41:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/testing/ptest (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/testing/ptest (from $GOPATH)
sdks/go/test/integration/driver/driver.go:26:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam (from $GOPATH)
sdks/go/test/integration/driver/driver.go:27:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs (from $GOPATH)
sdks/go/test/integration/driver/driver.go:28:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/log" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/log (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/log (from $GOPATH)
sdks/go/test/integration/driver/driver.go:29:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/x/beamx" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/x/beamx (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/x/beamx (from $GOPATH)
sdks/go/test/integration/driver/driver.go:30:2: cannot find package "github.com/apache/beam/sdks/go/test/integration/primitives" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/primitives (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/primitives (from $GOPATH)
sdks/go/test/integration/driver/driver.go:31:2: cannot find package "github.com/apache/beam/sdks/go/test/integration/synthetic" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/synthetic (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/synthetic (from $GOPATH)
sdks/go/test/integration/driver/driver.go:32:2: cannot find package "github.com/apache/beam/sdks/go/test/integration/wordcount" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/wordcount (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/wordcount (from $GOPATH)
sdks/go/test/integration/primitives/cogbk.go:22:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/testing/passert" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/testing/passert (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/testing/passert (from $GOPATH)
sdks/go/test/integration/synthetic/synthetic.go:21:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/io/synthetic" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/synthetic (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/synthetic (from $GOPATH)
sdks/go/test/integration/wordcount/wordcount.go:27:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/io/textio" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio (from $GOPATH)
sdks/go/test/integration/wordcount/wordcount.go:29:2: cannot find package "github.com/apache/beam/sdks/go/pkg/beam/transforms/stats" in any of:
	/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/transforms/stats (from $GOROOT)
	/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/transforms/stats (from $GOPATH)

if [[ "$RUNNER" == "dataflow" ]]; then
  # Delete the container locally and remotely
  docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
  gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"

  # Clean up tempdir
  rm -rf $TMPDIR
fi

exit $TEST_EXIT_CODE
exit_background_processes

> Task :sdks:go:test:flinkValidatesRunner FAILED

FAILURE: Build failed with an exception.

* Where:
Build file '<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src/sdks/go/test/build.gradle'> line: 86

* What went wrong:
Execution failed for task ':sdks:go:test:flinkValidatesRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1

* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.

* Get more help at https://help.gradle.org

Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.8/userguide/command_line_interface.html#sec:command_line_warnings

BUILD FAILED in 12m 0s
139 actionable tasks: 101 executed, 36 from cache, 2 up-to-date
Gradle was unable to watch the file system for changes. The inotify watches limit is too low.

Publishing build scan...
https://gradle.com/s/dmbt2nh5ana54

Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure

---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org