You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mg...@apache.org on 2024/08/12 04:25:51 UTC
(spark-connect-go) branch master updated: [SPARK-48754] Adding Integration Tests
This is an automated email from the ASF dual-hosted git repository.
mgrund pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-go.git
The following commit(s) were added to refs/heads/master by this push:
new 0ed4200 [SPARK-48754] Adding Integration Tests
0ed4200 is described below
commit 0ed42000396301279e3fdda466fd92dc7002f9d3
Author: Martin Grund <ma...@databricks.com>
AuthorDate: Mon Aug 12 06:25:42 2024 +0200
[SPARK-48754] Adding Integration Tests
### What changes were proposed in this pull request?
This patch adds a new integration test target that allows running Spark Connect Go integration tests against a locally running Spark version. This makes sure that the project is not only setting up the mocks correctly, but works against a real Spark cluster.
In addition, this PR modifies the build and test workflow to:
* Run the integration tests.
* Calculate the code coverage across unit and integration test.
* Fail the PR if the code coverage is < 60%. This value can be changed later on when the overall coverage increases.
### Why are the changes needed?
Compatibility.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Integration using GH Actions.
Closes #40 from grundprinzip/integration.
Authored-by: Martin Grund <ma...@databricks.com>
Signed-off-by: Martin Grund <ma...@databricks.com>
---
.github/workflows/build.yml | 49 ++++++++++++++++++++
.gitignore | 5 ++-
CONTRIBUTING.md | 22 +++++++++
Makefile | 14 +++---
internal/tests/integration/spark_runner.go | 72 ++++++++++++++++++++++++++++++
internal/tests/integration/sql_test.go | 62 +++++++++++++++++++++++++
spark/client/channel/channel.go | 3 +-
spark/sparkerrors/errors.go | 5 +++
8 files changed, 223 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 706745a..53c21e6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -31,6 +31,10 @@ on:
branches:
- master
+env:
+ SPARK_VERSION: '3.5.1'
+ HADOOP_VERSION: '3'
+
permissions:
# Required: allow read access to the content for analysis.
contents: read
@@ -53,13 +57,58 @@ jobs:
name: Setup Go
with:
go-version-file: 'go.mod'
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: zulu
+
+ - name: Cache Spark Installation
+ uses: actions/cache@v2
+ with:
+ key: spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}
+ path: |
+ $GITHUB_WORKSPACE/deps/spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}
+
+ - name: Setup Apache Spark
+ if: steps.cache.outputs.cache-hit != 'true'
+ run: |
+ echo "Apache Spark is not installed"
+ # Access the directory.
+ mkdir -p $GITHUB_WORKSPACE/deps/
+ cd $GITHUB_WORKSPACE/deps/
+ wget -q https://dlcdn.apache.org/spark/spark-${{ env.SPARK_VERSION }}/spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}.tgz
+ tar -xzf spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}.tgz
+ # Delete the old file
+ rm spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}.tgz
+
+ # Setup the Environment Variables
+ echo "Apache Spark is ready to use"
+ echo "SPARK_HOME=${GITHUB_WORKSPACE}/deps/spark-${{ env.SPARK_VERSION }}-bin-hadoop${{ env.HADOOP_VERSION }}" >> "$GITHUB_ENV"
+ cd $GITHUB_WORKSPACE
- name: Run Build & Test
run: |
go mod download -x
make gen
make
make test
+ - name: Run Integration Test
+ run: |
+ # make integration
+ make gen && make && make integration
+ - name: Run Code Coverage
+ run: |
+ make coverage
+ - uses: PaloAltoNetworks/cov@3.0.0
+ with:
+ cov_mode: coverage
+ main_branch: master
+ cov_threshold: 60
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
version: v1.59
+
diff --git a/.gitignore b/.gitignore
index 8381e8d..3f791f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ internal/generated.out
# Ignore Coverage Files
coverage*
+cov.report
# Ignore IDE files
.idea/
@@ -29,4 +30,6 @@ cmd/spark-connect-example-raw-grpc-client/spark-connect-example-raw-grpc-client
cmd/spark-connect-example-spark-session/spark-connect-example-spark-session
target
-lib
\ No newline at end of file
+lib
+
+deps
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4f11da2..e228fbf 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,6 +31,28 @@ This requires the following tools to be present in your PATH:
2. `gofumpt` for formatting Go code
3. `golangci-lint` for linting Go code
+### Running Tests
+
+To run the tests locally, you can run:
+
+```bash
+make test
+```
+
+This will run the unit tests. If you want to run the integration tests, you can run:
+
+```bash
+make integration
+```
+
+Lastly, if you want to run all tests and generate the coverage analysis, you can run:
+
+```bash
+make fulltest
+```
+
+The output of the coverage analysis will be in the `coverage.out` file. An HTML version of
+the coverage report is generated and accessible at `coverage.html`.
### How to write tests
diff --git a/Makefile b/Makefile
index c2c1d31..999fea5 100644
--- a/Makefile
+++ b/Makefile
@@ -92,14 +92,14 @@ test: $(BUILD_OUTPUT)
@echo -n " ";\
$(GO) test -v -run '(Test|Example)' $(BUILDFLAGS) $(TESTFLAGS) $(pkg) || exit 1)
-fulltest: $(BUILD_OUTPUT)
+coverage: $(BUILD_OUTPUT)
@echo ">> TEST, \"coverage\""
- @echo "mode: atomic" > coverage-all.out
- @$(foreach pkg, $(PKGS),\
- echo -n " ";\
- go test -run '(Test|Example)' $(BUILDFLAGS) $(TESTFLAGS) -coverprofile=coverage.out -covermode=atomic $(pkg) || exit 1;\
- tail -n +2 coverage.out >> coverage-all.out;)
- @$(GO) tool cover -html=coverage-all.out -o coverage-all.html
+ @$(GO) test -cover -coverprofile=coverage.out -covermode=atomic -coverpkg=./spark/...,./internal/tests/... ./spark/... ./internal/tests/...
+ @$(GO) tool cover -html=coverage.out -o coverage.html
+
+integration: $(BUILD_OUTPUT)
+ @echo ">> TEST, \"integration\""
+ @$(GO) test ./internal/tests/...
check:
@echo -n ">> CHECK"
diff --git a/internal/tests/integration/spark_runner.go b/internal/tests/integration/spark_runner.go
new file mode 100644
index 0000000..86a8881
--- /dev/null
+++ b/internal/tests/integration/spark_runner.go
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+ "fmt"
+ "io"
+ "net"
+ "os"
+ "os/exec"
+ "time"
+
+ "github.com/apache/spark-connect-go/v35/spark/sparkerrors"
+)
+
+func StartSparkConnect() (int64, error) {
+ sparkHome := os.Getenv("SPARK_HOME")
+ if sparkHome == "" {
+ return -1, sparkerrors.WithString(sparkerrors.TestSetupError, "SPARK_HOME not set")
+ }
+
+ fmt.Printf("Starting Spark Connect Server in: %v\n", os.Getenv("SPARK_HOME"))
+
+ cmd := exec.Command("./sbin/start-connect-server.sh", "--wait", "--conf",
+ "spark.log.structuredLogging.enabled=false", "--packages",
+ "org.apache.spark:spark-connect_2.12:3.5.1")
+ cmd.Dir = sparkHome
+
+ stdout, _ := cmd.StdoutPipe()
+ if err := cmd.Start(); err != nil {
+ return -1, sparkerrors.WithType(sparkerrors.TestSetupError, err)
+ }
+
+ timeout := time.After(60 * time.Second)
+ tick := time.NewTicker(1 * time.Second)
+
+ for {
+ select {
+ case <-timeout:
+ out, _ := io.ReadAll(stdout)
+ fmt.Printf("Output: %v\n", string(out))
+ return -1, sparkerrors.WithString(sparkerrors.TestSetupError,
+ "timeout waiting for Spark Connect to start")
+ case <-tick.C:
+ if cmd.ProcessState != nil && cmd.ProcessState.Exited() {
+ return -1, sparkerrors.WithString(sparkerrors.TestSetupError, "Spark Connect exited")
+ }
+ conn, err := net.Dial("tcp", "localhost:15002")
+ if err == nil {
+ conn.Close()
+ return int64(cmd.Process.Pid), nil
+ }
+ }
+ }
+}
+
+func StopSparkConnect(pid int64) error {
+ return nil
+}
diff --git a/internal/tests/integration/sql_test.go b/internal/tests/integration/sql_test.go
new file mode 100644
index 0000000..5f4435a
--- /dev/null
+++ b/internal/tests/integration/sql_test.go
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+ "context"
+ "log"
+ "os"
+ "testing"
+
+ "github.com/apache/spark-connect-go/v35/spark/sql"
+ "github.com/apache/spark-connect-go/v35/spark/sql/functions"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIntegration_RunSQLCommand(t *testing.T) {
+ // Run SQL command
+ ctx := context.Background()
+ spark, err := sql.NewSessionBuilder().Remote("sc://localhost").Build(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ df, err := spark.Sql(ctx, "select * from range(100)")
+ assert.NoError(t, err)
+ res, err := df.Collect(ctx)
+ assert.NoError(t, err)
+ assert.Equal(t, 100, len(res))
+
+ col, err := df.Col("id")
+ assert.NoError(t, err)
+ df, err = df.Filter(col.Lt(functions.Lit(10)))
+ assert.NoError(t, err)
+ res, err = df.Collect(ctx)
+ assert.NoErrorf(t, err, "Must be able to collect the rows.")
+ assert.Equal(t, 10, len(res))
+}
+
+func TestMain(m *testing.M) {
+ pid, err := StartSparkConnect()
+ if err != nil {
+ log.Fatal(err)
+ }
+ code := m.Run()
+ if err = StopSparkConnect(pid); err != nil {
+ log.Fatal(err)
+ }
+ os.Exit(code)
+}
diff --git a/spark/client/channel/channel.go b/spark/client/channel/channel.go
index 3d2e246..c8f9b70 100644
--- a/spark/client/channel/channel.go
+++ b/spark/client/channel/channel.go
@@ -112,7 +112,8 @@ func (cb *BaseBuilder) Build(ctx context.Context) (*grpc.ClientConn, error) {
remote := fmt.Sprintf("%v:%v", cb.host, cb.port)
conn, err := grpc.NewClient(remote, opts...)
if err != nil {
- return nil, sparkerrors.WithType(fmt.Errorf("failed to connect to remote %s: %w", remote, err), sparkerrors.ConnectionError)
+ return nil, sparkerrors.WithType(fmt.Errorf("failed to connect to remote %s: %w",
+ remote, err), sparkerrors.ConnectionError)
}
return conn, nil
}
diff --git a/spark/sparkerrors/errors.go b/spark/sparkerrors/errors.go
index a7f5bef..75c693d 100644
--- a/spark/sparkerrors/errors.go
+++ b/spark/sparkerrors/errors.go
@@ -44,6 +44,10 @@ func WithType(err error, errType errorType) error {
return &wrappedError{cause: err, errorType: errType}
}
+func WithString(err error, errMsg string) error {
+ return &wrappedError{cause: err, errorType: errors.New(errMsg)}
+}
+
type errorType error
var (
@@ -54,6 +58,7 @@ var (
InvalidPlanError = errorType(errors.New("invalid plan"))
RetriesExceeded = errorType(errors.New("retries exceeded"))
InvalidServerSideSessionError = errorType(errors.New("invalid server side session"))
+ TestSetupError = errorType(errors.New("test setup error"))
)
type UnsupportedResponseTypeError struct {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org