You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mg...@apache.org on 2024/10/25 12:20:27 UTC
(spark-connect-go) branch master updated: add option function to spark read
This is an automated email from the ASF dual-hosted git repository.
mgrund pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-go.git
The following commit(s) were added to refs/heads/master by this push:
new a8fccbf add option function to spark read
a8fccbf is described below
commit a8fccbf4619bb1a59fe87d9a1fcd7361e26142ab
Author: Amin Karkhi <am...@digikala.com>
AuthorDate: Fri Oct 25 14:20:17 2024 +0200
add option function to spark read
### What changes were proposed in this pull request?
i wanted to add jdbc driver so i could read from mysql from spark. the option function was not supported in the spark read so i have added this function. now we can use spark.read().option(key,value)
### Why are the changes needed?
using jdbc mostly needs this feature
### Does this PR introduce _any_ user-facing change?
no but it will add more functionality to the user
### How was this patch tested?
i added a simple test and used it to connect to a mysql server. i just tested conectivity but i had to use this command when starting spark:
```sh
➜ sbin ./start-connect-server.sh --packages org.apache.spark:spark-connect_2.13:3.5.2,org.mariadb-java-client:3.3.2
'''
note: integrity test fails because of my java version.
Closes #78 from amkarkhi/master.
Lead-authored-by: Amin Karkhi <am...@digikala.com>
Co-authored-by: Amin Karkhi <am...@gmail.com>
Signed-off-by: Martin Grund <ma...@databricks.com>
---
internal/tests/integration/dataframe_test.go | 28 ++++++++++++++++++++++++++++
spark/sql/dataframereader.go | 15 ++++++++++++++-
spark/sql/dataframereader_test.go | 13 +++++++++++++
spark/sql/plan.go | 16 ++++++++++++++++
4 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/internal/tests/integration/dataframe_test.go b/internal/tests/integration/dataframe_test.go
index ae9e06b..b04b7fe 100644
--- a/internal/tests/integration/dataframe_test.go
+++ b/internal/tests/integration/dataframe_test.go
@@ -17,6 +17,8 @@ package integration
import (
"context"
+ "fmt"
+ "os"
"testing"
"github.com/apache/spark-connect-go/v35/spark/sql/utils"
@@ -694,3 +696,29 @@ func TestDataFrame_FreqItems(t *testing.T) {
assert.NoErrorf(t, err, "%+v", err)
assert.Len(t, res, 1)
}
+
+func TestDataFrame_WithOption(t *testing.T) {
+ ctx, spark := connect()
+ file, err := os.CreateTemp("", "example")
+ defer os.Remove(file.Name())
+ assert.NoError(t, err)
+ defer file.Close()
+ _, err = file.WriteString("id#name,name\n")
+ assert.NoError(t, err)
+ for i := 0; i < 10; i++ {
+ _, err = file.WriteString(fmt.Sprintf("%d#alice,alice\n", i))
+ assert.NoError(t, err)
+ }
+ df, err := spark.Read().Format("csv").
+ Option("header", "true").
+ Option("quote", "\"").
+ Option("sep", "#").
+ Option("escapeQuotes", "true").
+ // Option("skipLines", "5"). //TODO: this needs more insight
+ Option("inferSchema", "false").
+ Load(file.Name())
+ assert.NoError(t, err)
+ c, err := df.Count(ctx)
+ assert.NoError(t, err)
+ assert.Equal(t, int64(10), c)
+}
diff --git a/spark/sql/dataframereader.go b/spark/sql/dataframereader.go
index 5ca3a33..636435a 100644
--- a/spark/sql/dataframereader.go
+++ b/spark/sql/dataframereader.go
@@ -26,12 +26,14 @@ type DataFrameReader interface {
Load(path string) (DataFrame, error)
// Reads a table from the underlying data source.
Table(name string) (DataFrame, error)
+ Option(key, value string) DataFrameReader
}
// dataFrameReaderImpl is an implementation of DataFrameReader interface.
type dataFrameReaderImpl struct {
sparkSession *sparkSessionImpl
formatSource string
+ options map[string]string
}
// NewDataframeReader creates a new DataFrameReader
@@ -55,5 +57,16 @@ func (w *dataFrameReaderImpl) Load(path string) (DataFrame, error) {
if w.formatSource != "" {
format = w.formatSource
}
- return NewDataFrame(w.sparkSession, newReadWithFormatAndPath(path, format)), nil
+ if w.options == nil {
+ return NewDataFrame(w.sparkSession, newReadWithFormatAndPath(path, format)), nil
+ }
+ return NewDataFrame(w.sparkSession, newReadWithFormatAndPathAndOptions(path, format, w.options)), nil
+}
+
+func (w *dataFrameReaderImpl) Option(key, value string) DataFrameReader {
+ if w.options == nil {
+ w.options = make(map[string]string)
+ }
+ w.options[key] = value
+ return w
}
diff --git a/spark/sql/dataframereader_test.go b/spark/sql/dataframereader_test.go
index 8b17794..2971e90 100644
--- a/spark/sql/dataframereader_test.go
+++ b/spark/sql/dataframereader_test.go
@@ -39,3 +39,16 @@ func TestRelationContainsPathAndFormat(t *testing.T) {
assert.Equal(t, &formatSource, relation.GetRead().GetDataSource().Format)
assert.Equal(t, path, relation.GetRead().GetDataSource().Paths[0])
}
+
+func TestRelationContainsPathAndFormatAndOptions(t *testing.T) {
+ formatSource := "source"
+ path := "path"
+ options := map[string]string{"key": "value"}
+ relation := newReadWithFormatAndPathAndOptions(path, formatSource, options)
+ assert.NotNil(t, relation)
+ assert.Equal(t, &formatSource, relation.GetRead().GetDataSource().Format)
+ assert.Equal(t, path, relation.GetRead().GetDataSource().Paths[0])
+ for i, v := range options {
+ assert.Equal(t, v, relation.GetRead().GetDataSource().Options[i])
+ }
+}
diff --git a/spark/sql/plan.go b/spark/sql/plan.go
index 8ecacab..7debf86 100644
--- a/spark/sql/plan.go
+++ b/spark/sql/plan.go
@@ -64,3 +64,19 @@ func newReadWithFormatAndPath(path, format string) *proto.Relation {
},
}
}
+
+func newReadWithFormatAndPathAndOptions(path, format string, options map[string]string) *proto.Relation {
+ return &proto.Relation{
+ RelType: &proto.Relation_Read{
+ Read: &proto.Read{
+ ReadType: &proto.Read_DataSource_{
+ DataSource: &proto.Read_DataSource{
+ Format: &format,
+ Paths: []string{path},
+ Options: options,
+ },
+ },
+ },
+ },
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org