You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mg...@apache.org on 2024/10/25 12:20:27 UTC

(spark-connect-go) branch master updated: add option function to spark read

This is an automated email from the ASF dual-hosted git repository.

mgrund pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-go.git


The following commit(s) were added to refs/heads/master by this push:
     new a8fccbf  add option function to spark read
a8fccbf is described below

commit a8fccbf4619bb1a59fe87d9a1fcd7361e26142ab
Author: Amin Karkhi <am...@digikala.com>
AuthorDate: Fri Oct 25 14:20:17 2024 +0200

    add option function to spark read
    
    ### What changes were proposed in this pull request?
    i wanted to add jdbc driver so i could read from mysql from spark. the option function was not supported in the spark read so i have added this function. now we can use spark.read().option(key,value)
    
    ### Why are the changes needed?
    using jdbc mostly needs this feature
    
    ### Does this PR introduce _any_ user-facing change?
    no but it will add more functionality to the user
    
    ### How was this patch tested?
    i added a simple test and used it to connect to a mysql server. i just tested conectivity but i had to use this command when starting spark:
    ```sh
    ➜  sbin ./start-connect-server.sh --packages org.apache.spark:spark-connect_2.13:3.5.2,org.mariadb-java-client:3.3.2
    '''
    note: integrity test fails because of my java version.
    
    Closes #78 from amkarkhi/master.
    
    Lead-authored-by: Amin Karkhi <am...@digikala.com>
    Co-authored-by: Amin Karkhi <am...@gmail.com>
    Signed-off-by: Martin Grund <ma...@databricks.com>
---
 internal/tests/integration/dataframe_test.go | 28 ++++++++++++++++++++++++++++
 spark/sql/dataframereader.go                 | 15 ++++++++++++++-
 spark/sql/dataframereader_test.go            | 13 +++++++++++++
 spark/sql/plan.go                            | 16 ++++++++++++++++
 4 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/internal/tests/integration/dataframe_test.go b/internal/tests/integration/dataframe_test.go
index ae9e06b..b04b7fe 100644
--- a/internal/tests/integration/dataframe_test.go
+++ b/internal/tests/integration/dataframe_test.go
@@ -17,6 +17,8 @@ package integration
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"testing"
 
 	"github.com/apache/spark-connect-go/v35/spark/sql/utils"
@@ -694,3 +696,29 @@ func TestDataFrame_FreqItems(t *testing.T) {
 	assert.NoErrorf(t, err, "%+v", err)
 	assert.Len(t, res, 1)
 }
+
+func TestDataFrame_WithOption(t *testing.T) {
+	ctx, spark := connect()
+	file, err := os.CreateTemp("", "example")
+	defer os.Remove(file.Name())
+	assert.NoError(t, err)
+	defer file.Close()
+	_, err = file.WriteString("id#name,name\n")
+	assert.NoError(t, err)
+	for i := 0; i < 10; i++ {
+		_, err = file.WriteString(fmt.Sprintf("%d#alice,alice\n", i))
+		assert.NoError(t, err)
+	}
+	df, err := spark.Read().Format("csv").
+		Option("header", "true").
+		Option("quote", "\"").
+		Option("sep", "#").
+		Option("escapeQuotes", "true").
+		// Option("skipLines", "5"). //TODO: this needs more insight
+		Option("inferSchema", "false").
+		Load(file.Name())
+	assert.NoError(t, err)
+	c, err := df.Count(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, int64(10), c)
+}
diff --git a/spark/sql/dataframereader.go b/spark/sql/dataframereader.go
index 5ca3a33..636435a 100644
--- a/spark/sql/dataframereader.go
+++ b/spark/sql/dataframereader.go
@@ -26,12 +26,14 @@ type DataFrameReader interface {
 	Load(path string) (DataFrame, error)
 	// Reads a table from the underlying data source.
 	Table(name string) (DataFrame, error)
+	Option(key, value string) DataFrameReader
 }
 
 // dataFrameReaderImpl is an implementation of DataFrameReader interface.
 type dataFrameReaderImpl struct {
 	sparkSession *sparkSessionImpl
 	formatSource string
+	options      map[string]string
 }
 
 // NewDataframeReader creates a new DataFrameReader
@@ -55,5 +57,16 @@ func (w *dataFrameReaderImpl) Load(path string) (DataFrame, error) {
 	if w.formatSource != "" {
 		format = w.formatSource
 	}
-	return NewDataFrame(w.sparkSession, newReadWithFormatAndPath(path, format)), nil
+	if w.options == nil {
+		return NewDataFrame(w.sparkSession, newReadWithFormatAndPath(path, format)), nil
+	}
+	return NewDataFrame(w.sparkSession, newReadWithFormatAndPathAndOptions(path, format, w.options)), nil
+}
+
+func (w *dataFrameReaderImpl) Option(key, value string) DataFrameReader {
+	if w.options == nil {
+		w.options = make(map[string]string)
+	}
+	w.options[key] = value
+	return w
 }
diff --git a/spark/sql/dataframereader_test.go b/spark/sql/dataframereader_test.go
index 8b17794..2971e90 100644
--- a/spark/sql/dataframereader_test.go
+++ b/spark/sql/dataframereader_test.go
@@ -39,3 +39,16 @@ func TestRelationContainsPathAndFormat(t *testing.T) {
 	assert.Equal(t, &formatSource, relation.GetRead().GetDataSource().Format)
 	assert.Equal(t, path, relation.GetRead().GetDataSource().Paths[0])
 }
+
+func TestRelationContainsPathAndFormatAndOptions(t *testing.T) {
+	formatSource := "source"
+	path := "path"
+	options := map[string]string{"key": "value"}
+	relation := newReadWithFormatAndPathAndOptions(path, formatSource, options)
+	assert.NotNil(t, relation)
+	assert.Equal(t, &formatSource, relation.GetRead().GetDataSource().Format)
+	assert.Equal(t, path, relation.GetRead().GetDataSource().Paths[0])
+	for i, v := range options {
+		assert.Equal(t, v, relation.GetRead().GetDataSource().Options[i])
+	}
+}
diff --git a/spark/sql/plan.go b/spark/sql/plan.go
index 8ecacab..7debf86 100644
--- a/spark/sql/plan.go
+++ b/spark/sql/plan.go
@@ -64,3 +64,19 @@ func newReadWithFormatAndPath(path, format string) *proto.Relation {
 		},
 	}
 }
+
+func newReadWithFormatAndPathAndOptions(path, format string, options map[string]string) *proto.Relation {
+	return &proto.Relation{
+		RelType: &proto.Relation_Read{
+			Read: &proto.Read{
+				ReadType: &proto.Read_DataSource_{
+					DataSource: &proto.Read_DataSource{
+						Format:  &format,
+						Paths:   []string{path},
+						Options: options,
+					},
+				},
+			},
+		},
+	}
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org