You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/08/08 17:16:41 UTC
(iceberg-go) branch main updated: fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 03bd440 fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)
03bd440 is described below
commit 03bd4401cb14a4ca392c8556ec301a87ec51c8c7
Author: Leon Lin <li...@gmail.com>
AuthorDate: Fri Aug 8 10:16:36 2025 -0700
fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)
### Description
* Currently iceberg spark integration tests are not enabled in ci
pipeline
* Update local spark to execute script with argument to validate
producer scenarios.
---
.github/workflows/go-integration.yml | 13 +-
internal/recipe/Dockerfile | 4 +-
internal/recipe/local_spark.go | 80 +++++++++-
internal/recipe/run_spark_count_sql.py | 22 ---
internal/recipe/validation.py | 56 +++++++
table/transaction_test.go | 258 +++++++++++++++++++++++++++++++--
6 files changed, 393 insertions(+), 40 deletions(-)
diff --git a/.github/workflows/go-integration.yml b/.github/workflows/go-integration.yml
index b90ccb8..d8d9442 100644
--- a/.github/workflows/go-integration.yml
+++ b/.github/workflows/go-integration.yml
@@ -55,9 +55,11 @@ jobs:
docker compose -f internal/recipe/docker-compose.yml exec -T spark-iceberg ipython ./provision.py
sleep 10
- - name: Get minio container IP
+ - name: Setup environment variables
run: |
echo "AWS_S3_ENDPOINT=http://$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' minio):9000" >> $GITHUB_ENV
+ echo "SPARK_CONTAINER_ID=$(docker ps -qf 'name=spark-iceberg')" >> $GITHUB_ENV
+ echo "DOCKER_API_VERSION=$(docker version -f '{{.Server.APIVersion}}')" >> $GITHUB_ENV
- name: Run integration tests
env:
@@ -68,6 +70,15 @@ jobs:
go test -tags integration -v ./io
go test -tags integration -v -run="^TestRestIntegration$" ./catalog/rest
+ - name: Run spark integration tests
+ env:
+ AWS_S3_ENDPOINT: "${{ env.AWS_S3_ENDPOINT }}"
+ AWS_REGION: "us-east-1"
+ SPARK_CONTAINER_ID: "${{ env.SPARK_CONTAINER_ID }}"
+ DOCKER_API_VER: "${{ env.DOCKER_API_VER }}"
+ run: |
+ go test -tags integration -v -run="^TestSparkIntegration" ./table
+
- name: Show debug logs
if: ${{ failure() }}
run: docker compose -f internal/recipe/docker-compose.yml logs
diff --git a/internal/recipe/Dockerfile b/internal/recipe/Dockerfile
index 172d2aa..a5d491c 100644
--- a/internal/recipe/Dockerfile
+++ b/internal/recipe/Dockerfile
@@ -20,7 +20,7 @@ RUN pip3 install pyiceberg[s3fs,hive]
RUN pip3 install pyarrow
COPY provision.py .
-COPY run_spark_count_sql.py .
+COPY validation.py .
ENTRYPOINT ["./entrypoint.sh"]
-CMD ["notebook"]
\ No newline at end of file
+CMD ["notebook"]
diff --git a/internal/recipe/local_spark.go b/internal/recipe/local_spark.go
index e5ecd47..ca07141 100644
--- a/internal/recipe/local_spark.go
+++ b/internal/recipe/local_spark.go
@@ -25,12 +25,17 @@ import (
"os"
"testing"
+ "github.com/docker/docker/api/types/container"
+ "github.com/docker/docker/api/types/filters"
+ "github.com/docker/docker/client"
"github.com/testcontainers/testcontainers-go/modules/compose"
)
//go:embed docker-compose.yml
var composeFile []byte
+const sparkContainer = "spark-iceberg"
+
func Start(t *testing.T) (*compose.DockerCompose, error) {
if _, ok := os.LookupEnv("AWS_S3_ENDPOINT"); ok {
return nil, nil
@@ -44,7 +49,7 @@ func Start(t *testing.T) (*compose.DockerCompose, error) {
if err := stack.Up(t.Context()); err != nil {
return stack, fmt.Errorf("fail to up compose: %w", err)
}
- spark, err := stack.ServiceContainer(t.Context(), "spark-iceberg")
+ spark, err := stack.ServiceContainer(t.Context(), sparkContainer)
if err != nil {
return stack, fmt.Errorf("fail to find spark-iceberg: %w", err)
}
@@ -62,3 +67,76 @@ func Start(t *testing.T) (*compose.DockerCompose, error) {
return stack, nil
}
+
+func ExecuteSpark(t *testing.T, scriptPath string, args ...string) (string, error) {
+ var cli *client.Client
+ var err error
+
+ if apiVersion, ok := os.LookupEnv("DOCKER_API_VERSION"); ok {
+ cli, err = client.NewClientWithOpts(
+ client.FromEnv,
+ client.WithVersion(apiVersion),
+ )
+ } else {
+ cli, err = client.NewClientWithOpts(
+ client.FromEnv,
+ )
+ }
+ if err != nil {
+ return "", err
+ }
+ defer func(cli *client.Client) {
+ err := cli.Close()
+ if err != nil {
+ t.Logf("failed to close docker client")
+ }
+ }(cli)
+
+ var sparkContainerID string
+ if _, ok := os.LookupEnv("SPARK_CONTAINER_ID"); ok {
+ sparkContainerID = os.Getenv("SPARK_CONTAINER_ID")
+ } else {
+ filter := filters.NewArgs(filters.Arg("name", sparkContainer))
+ containers, err := cli.ContainerList(t.Context(), container.ListOptions{
+ Filters: filter,
+ })
+ if err != nil {
+ return "", err
+ }
+ if len(containers) != 1 {
+ return "", fmt.Errorf("unable to find container: %s", sparkContainer)
+ }
+ sparkContainerID = containers[0].ID
+ }
+
+ response, err := cli.ContainerExecCreate(t.Context(), sparkContainerID, container.ExecOptions{
+ Cmd: append([]string{"python", scriptPath}, args...),
+ AttachStdout: true,
+ AttachStderr: true,
+ })
+ if err != nil {
+ return "", err
+ }
+
+ attachResp, err := cli.ContainerExecAttach(t.Context(), response.ID, container.ExecAttachOptions{})
+ if err != nil {
+ return "", err
+ }
+ defer attachResp.Close()
+
+ output, err := io.ReadAll(attachResp.Reader)
+ if err != nil {
+ return "", err
+ }
+ fmt.Printf("%s\n", string(output))
+
+ inspect, err := cli.ContainerExecInspect(t.Context(), response.ID)
+ if err != nil {
+ return "", err
+ }
+ if inspect.ExitCode != 0 {
+ return "", fmt.Errorf("failed to execute script with exit code: %d", inspect.ExitCode)
+ }
+
+ return string(output), nil
+}
diff --git a/internal/recipe/run_spark_count_sql.py b/internal/recipe/run_spark_count_sql.py
deleted file mode 100644
index 9c472ff..0000000
--- a/internal/recipe/run_spark_count_sql.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyspark.sql import SparkSession
-
-spark = SparkSession.builder.getOrCreate()
-
-spark.sql("SELECT COUNT(*) FROM default.test_partitioned_by_days").show()
diff --git a/internal/recipe/validation.py b/internal/recipe/validation.py
new file mode 100644
index 0000000..1ac9421
--- /dev/null
+++ b/internal/recipe/validation.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.getOrCreate()
+
+
+def testSetProperties():
+ spark.sql("SHOW TBLPROPERTIES default.go_test_set_properties").show(truncate=False)
+
+
+def testAddedFile():
+ spark.sql("SELECT COUNT(*) FROM default.test_partitioned_by_days").show(truncate=False)
+
+
+def testReadDifferentDataTypes():
+ spark.sql("DESCRIBE TABLE EXTENDED default.go_test_different_data_types").show(truncate=False)
+ spark.sql("SELECT * FROM default.go_test_different_data_types").show(truncate=False)
+
+
+def testReadSpecUpdate():
+ spark.sql("DESCRIBE TABLE EXTENDED default.go_test_update_spec").show(truncate=False)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--test", type=str, required=True, help="Name of the test to run")
+ args = parser.parse_args()
+
+ if args.test == "TestSetProperties":
+ testSetProperties()
+
+ if args.test == "TestAddedFile":
+ testAddedFile()
+
+ if args.test == "TestReadDifferentDataTypes":
+ testReadDifferentDataTypes()
+
+ if args.test == "TestReadSpecUpdate":
+ testReadSpecUpdate()
diff --git a/table/transaction_test.go b/table/transaction_test.go
index 4f68bed..e968eaf 100644
--- a/table/transaction_test.go
+++ b/table/transaction_test.go
@@ -21,7 +21,6 @@ package table_test
import (
"context"
- "io"
"strings"
"testing"
"time"
@@ -54,9 +53,6 @@ func (s *SparkIntegrationTestSuite) SetupSuite() {
var err error
s.stack, err = recipe.Start(s.T())
s.Require().NoError(err)
- if s.stack == nil {
- s.T().Skip("skipping test, AWS_S3_ENDPOINT is set")
- }
}
func (s *SparkIntegrationTestSuite) SetupTest() {
@@ -73,6 +69,51 @@ func (s *SparkIntegrationTestSuite) SetupTest() {
s.cat = cat
}
+func (s *SparkIntegrationTestSuite) TestSetProperties() {
+ icebergSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.Bool},
+ iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.String},
+ iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Int32},
+ )
+
+ tbl, err := s.cat.CreateTable(s.ctx, catalog.ToIdentifier("default", "go_test_set_properties"), icebergSchema)
+ s.Require().NoError(err)
+
+ tx := tbl.NewTransaction()
+
+ err = tx.SetProperties(iceberg.Properties{
+ table.ParquetCompressionKey: "snappy",
+ table.ManifestMergeEnabledKey: "true",
+ table.ManifestMinMergeCountKey: "1",
+ table.ManifestTargetSizeBytesKey: "1",
+ })
+ s.Require().NoError(err)
+
+ _, err = tx.Commit(s.ctx)
+ s.Require().NoError(err)
+
+ expectedOutput := `
++----------------------------------+---------------+
+|key |value |
++----------------------------------+---------------+
+|commit.manifest-merge.enabled |true |
+|commit.manifest.min-count-to-merge|1 |
+|commit.manifest.target-size-bytes |1 |
+|current-snapshot-id |none |
+|format |iceberg/parquet|
+|format-version |2 |
+|write.parquet.compression-codec |snappy |
++----------------------------------+---------------+
+`
+
+ output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestSetProperties")
+ s.Require().NoError(err)
+ s.Require().True(
+ strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+ "result does not contain expected output: %s", expectedOutput,
+ )
+}
+
func (s *SparkIntegrationTestSuite) TestAddFile() {
const filename = "s3://warehouse/default/test_partitioned_by_days/data/ts_day=2023-03-13/supertest.parquet"
@@ -114,21 +155,210 @@ func (s *SparkIntegrationTestSuite) TestAddFile() {
tbl, err = tx.Commit(s.ctx)
s.Require().NoError(err)
- spark, err := s.stack.ServiceContainer(s.T().Context(), "spark-iceberg")
+ expectedOutput := `
++--------+
+|count(1)|
++--------+
+|13 |
++--------+
+`
+
+ output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestAddedFile")
s.Require().NoError(err)
+ s.Require().True(
+ strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+ "result does not contain expected output: %s", expectedOutput,
+ )
+}
+
+func (s *SparkIntegrationTestSuite) TestDifferentDataTypes() {
+ icebergSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "bool", Type: iceberg.PrimitiveTypes.Bool},
+ iceberg.NestedField{ID: 2, Name: "string", Type: iceberg.PrimitiveTypes.String},
+ iceberg.NestedField{ID: 3, Name: "string_long", Type: iceberg.PrimitiveTypes.String},
+ iceberg.NestedField{ID: 4, Name: "int", Type: iceberg.PrimitiveTypes.Int32},
+ iceberg.NestedField{ID: 5, Name: "long", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 6, Name: "float", Type: iceberg.PrimitiveTypes.Float32},
+ iceberg.NestedField{ID: 7, Name: "double", Type: iceberg.PrimitiveTypes.Float64},
+ iceberg.NestedField{ID: 8, Name: "timestamp", Type: iceberg.PrimitiveTypes.Timestamp},
+ iceberg.NestedField{ID: 9, Name: "timestamptz", Type: iceberg.PrimitiveTypes.TimestampTz},
+ iceberg.NestedField{ID: 10, Name: "date", Type: iceberg.PrimitiveTypes.Date},
+ iceberg.NestedField{ID: 11, Name: "uuid", Type: iceberg.PrimitiveTypes.UUID},
+ iceberg.NestedField{ID: 12, Name: "binary", Type: iceberg.PrimitiveTypes.Binary},
+ iceberg.NestedField{ID: 13, Name: "fixed", Type: iceberg.FixedTypeOf(16)},
+ iceberg.NestedField{ID: 14, Name: "small_dec", Type: iceberg.DecimalTypeOf(8, 2)},
+ iceberg.NestedField{ID: 15, Name: "med_dec", Type: iceberg.DecimalTypeOf(16, 2)},
+ iceberg.NestedField{ID: 16, Name: "large_dec", Type: iceberg.DecimalTypeOf(24, 2)},
+ iceberg.NestedField{ID: 17, Name: "list", Type: &iceberg.ListType{
+ ElementID: 18,
+ Element: iceberg.PrimitiveTypes.Int32},
+ },
+ )
- _, stdout, err := spark.Exec(s.ctx, []string{"ipython", "./run_spark_count_sql.py"})
+ arrowSchema, err := table.SchemaToArrowSchema(icebergSchema, nil, true, false)
s.Require().NoError(err)
- output, err := io.ReadAll(stdout)
+ arrTable, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{
+ `[
+ {
+ "bool": false,
+ "string": "a",
+ "string_long": "` + strings.Repeat("a", 22) + `",
+ "int": 1,
+ "long": 1,
+ "float": 0.0,
+ "double": 0.0,
+ "timestamp": "2023-01-01T19:25:00.000000+08:00",
+ "timestamptz": "2023-01-01T19:25:00.000000Z",
+ "date": "2023-01-01",
+ "uuid": "00000000-0000-0000-0000-000000000000",
+ "binary": "AQ==",
+ "fixed": "AAAAAAAAAAAAAAAAAAAAAA==",
+ "small_dec": "123456.78",
+ "med_dec": "12345678901234.56",
+ "large_dec": "1234567890123456789012.34",
+ "list": [1, 2, 3]
+ },
+ {
+ "bool": null,
+ "string": null,
+ "string_long": null,
+ "int": null,
+ "long": null,
+ "float": null,
+ "double": null,
+ "timestamp": null,
+ "timestamptz": null,
+ "date": null,
+ "uuid": null,
+ "binary": null,
+ "fixed": null,
+ "small_dec": null,
+ "med_dec": null,
+ "large_dec": null,
+ "list": null
+ },
+ {
+ "bool": true,
+ "string": "z",
+ "string_long": "` + strings.Repeat("z", 22) + `",
+ "int": 9,
+ "long": 9,
+ "float": 0.9,
+ "double": 0.9,
+ "timestamp": "2023-03-01T19:25:00.000000+08:00",
+ "timestamptz": "2023-03-01T19:25:00.000000Z",
+ "date": "2023-03-01",
+ "uuid": "11111111-1111-1111-1111-111111111111",
+ "binary": "Eg==",
+ "fixed": "EREREREREREREREREREREQ==",
+ "small_dec": "876543.21",
+ "med_dec": "65432109876543.21",
+ "large_dec": "4321098765432109876543.21",
+ "list": [-1, -2, -3]
+ }
+ ]`,
+ })
s.Require().NoError(err)
- strings.HasSuffix(string(output), `
-+--------+
-|count(1)|
-+--------+
-| 13|
-+--------+
-`)
+ defer arrTable.Release()
+
+ tbl, err := s.cat.CreateTable(s.ctx, catalog.ToIdentifier("default", "go_test_different_data_types"), icebergSchema)
+ s.Require().NoError(err)
+
+ tx := tbl.NewTransaction()
+ err = tx.AppendTable(s.ctx, arrTable, 1, nil)
+ s.Require().NoError(err)
+
+ _, err = tx.Commit(s.ctx)
+ s.Require().NoError(err)
+
+ expectedSchema := `
++------------------+-------------+-------+
+|col_name |data_type |comment|
++------------------+-------------+-------+
+|bool |boolean |NULL |
+|string |string |NULL |
+|string_long |string |NULL |
+|int |int |NULL |
+|long |bigint |NULL |
+|float |float |NULL |
+|double |double |NULL |
+|timestamp |timestamp_ntz|NULL |
+|timestamptz |timestamp |NULL |
+|date |date |NULL |
+|uuid |string |NULL |
+|binary |binary |NULL |
+|fixed |binary |NULL |
+|small_dec |decimal(8,2) |NULL |
+|med_dec |decimal(16,2)|NULL |
+|large_dec |decimal(24,2)|NULL |
+|list |array<int> |NULL |
+`
+
+ expectedOutput := `
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+|bool |string|string_long |int |long|float|double|timestamp |timestamptz |date |uuid |binary|fixed |small_dec|med_dec |large_dec |list |
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+|false|a |aaaaaaaaaaaaaaaaaaaaaa|1 |1 |0.0 |0.0 |2023-01-01 11:25:00|2023-01-01 19:25:00|2023-01-01|00000000-0000-0000-0000-000000000000|[01] |[00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]|123456.78|12345678901234.56|1234567890123456789012.34|[1, 2, 3] |
+|NULL |NULL |NULL |NULL|NULL|NULL |NULL |NULL |NULL |NULL |NULL |NULL |NULL |NULL |NULL |NULL |NULL |
+|true |z |zzzzzzzzzzzzzzzzzzzzzz|9 |9 |0.9 |0.9 |2023-03-01 11:25:00|2023-03-01 19:25:00|2023-03-01|11111111-1111-1111-1111-111111111111|[12] |[11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11]|876543.21|65432109876543.21|4321098765432109876543.21|[-1, -2, -3]|
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+`
+
+ output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestReadDifferentDataTypes")
+ s.Require().NoError(err)
+ s.Require().True(
+ strings.Contains(strings.TrimSpace(output), strings.TrimSpace(expectedSchema)),
+ "result does not contain expected output: %s", expectedOutput,
+ )
+ s.Require().True(
+ strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+ "result does not contain expected output: %s", expectedOutput,
+ )
+}
+
+func (s *SparkIntegrationTestSuite) TestUpdateSpec() {
+ icebergSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.Bool},
+ iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.String},
+ iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Int32},
+ )
+
+ partitionSpec := iceberg.NewPartitionSpec(
+ iceberg.PartitionField{SourceID: 2, FieldID: 1000, Transform: iceberg.TruncateTransform{Width: 5}, Name: "bar_truncate"},
+ )
+
+ tbl, err := s.cat.CreateTable(
+ s.ctx,
+ catalog.ToIdentifier("default", "go_test_update_spec"),
+ icebergSchema,
+ catalog.WithPartitionSpec(&partitionSpec),
+ )
+ s.Require().NoError(err)
+
+ tx := tbl.NewTransaction()
+ err = tx.UpdateSpec(false).
+ AddField("baz", iceberg.BucketTransform{NumBuckets: 3}, "").
+ Commit()
+ s.Require().NoError(err)
+ _, err = tx.Commit(s.ctx)
+
+ partitionExpectedOutput := `
+|# Partitioning | | |
+|Part 0 |truncate(5, bar) | |
+|Part 1 |bucket(3, baz) | |
+| | | |
+`
+ metadataPartition := `
+|_partition |struct<bar_truncate:string,baz_bucket_3:int>| |
+`
+
+ output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestReadSpecUpdate")
+ s.Require().NoError(err)
+ s.Require().True(
+ strings.Contains(strings.TrimSpace(output), strings.TrimSpace(partitionExpectedOutput)),
+ "result does not contain expected output: %s", metadataPartition,
+ )
}
func TestSparkIntegration(t *testing.T) {