You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/08/08 17:16:41 UTC

(iceberg-go) branch main updated: fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 03bd440  fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)
03bd440 is described below

commit 03bd4401cb14a4ca392c8556ec301a87ec51c8c7
Author: Leon Lin <li...@gmail.com>
AuthorDate: Fri Aug 8 10:16:36 2025 -0700

    fix(infra/testing): Enable iceberg spark interops integration tests on ci pipeline (#493)
    
    ### Description
    * Currently iceberg spark integration tests are not enabled in ci
    pipeline
    * Update local spark to execute script with argument to validate
    producer scenarios.
---
 .github/workflows/go-integration.yml   |  13 +-
 internal/recipe/Dockerfile             |   4 +-
 internal/recipe/local_spark.go         |  80 +++++++++-
 internal/recipe/run_spark_count_sql.py |  22 ---
 internal/recipe/validation.py          |  56 +++++++
 table/transaction_test.go              | 258 +++++++++++++++++++++++++++++++--
 6 files changed, 393 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/go-integration.yml b/.github/workflows/go-integration.yml
index b90ccb8..d8d9442 100644
--- a/.github/workflows/go-integration.yml
+++ b/.github/workflows/go-integration.yml
@@ -55,9 +55,11 @@ jobs:
           docker compose -f internal/recipe/docker-compose.yml exec -T spark-iceberg ipython ./provision.py
           sleep 10
 
-      - name: Get minio container IP
+      - name: Setup environment variables
         run: |
           echo "AWS_S3_ENDPOINT=http://$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' minio):9000" >> $GITHUB_ENV
+          echo "SPARK_CONTAINER_ID=$(docker ps -qf 'name=spark-iceberg')" >> $GITHUB_ENV
+          echo "DOCKER_API_VERSION=$(docker version -f '{{.Server.APIVersion}}')" >> $GITHUB_ENV
 
       - name: Run integration tests
         env:
@@ -68,6 +70,15 @@ jobs:
           go test -tags integration -v ./io
           go test -tags integration -v -run="^TestRestIntegration$" ./catalog/rest
 
+      - name: Run spark integration tests
+        env:
+          AWS_S3_ENDPOINT: "${{ env.AWS_S3_ENDPOINT }}"
+          AWS_REGION: "us-east-1"
+          SPARK_CONTAINER_ID: "${{ env.SPARK_CONTAINER_ID }}"
+          DOCKER_API_VER: "${{ env.DOCKER_API_VER }}"
+        run: |
+          go test -tags integration -v -run="^TestSparkIntegration" ./table
+
       - name: Show debug logs
         if: ${{ failure() }}
         run: docker compose -f internal/recipe/docker-compose.yml logs
diff --git a/internal/recipe/Dockerfile b/internal/recipe/Dockerfile
index 172d2aa..a5d491c 100644
--- a/internal/recipe/Dockerfile
+++ b/internal/recipe/Dockerfile
@@ -20,7 +20,7 @@ RUN pip3 install pyiceberg[s3fs,hive]
 RUN pip3 install pyarrow
 
 COPY provision.py .
-COPY run_spark_count_sql.py .
+COPY validation.py .
 
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["notebook"]
\ No newline at end of file
+CMD ["notebook"]
diff --git a/internal/recipe/local_spark.go b/internal/recipe/local_spark.go
index e5ecd47..ca07141 100644
--- a/internal/recipe/local_spark.go
+++ b/internal/recipe/local_spark.go
@@ -25,12 +25,17 @@ import (
 	"os"
 	"testing"
 
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
 	"github.com/testcontainers/testcontainers-go/modules/compose"
 )
 
 //go:embed docker-compose.yml
 var composeFile []byte
 
+const sparkContainer = "spark-iceberg"
+
 func Start(t *testing.T) (*compose.DockerCompose, error) {
 	if _, ok := os.LookupEnv("AWS_S3_ENDPOINT"); ok {
 		return nil, nil
@@ -44,7 +49,7 @@ func Start(t *testing.T) (*compose.DockerCompose, error) {
 	if err := stack.Up(t.Context()); err != nil {
 		return stack, fmt.Errorf("fail to up compose: %w", err)
 	}
-	spark, err := stack.ServiceContainer(t.Context(), "spark-iceberg")
+	spark, err := stack.ServiceContainer(t.Context(), sparkContainer)
 	if err != nil {
 		return stack, fmt.Errorf("fail to find spark-iceberg: %w", err)
 	}
@@ -62,3 +67,76 @@ func Start(t *testing.T) (*compose.DockerCompose, error) {
 
 	return stack, nil
 }
+
+func ExecuteSpark(t *testing.T, scriptPath string, args ...string) (string, error) {
+	var cli *client.Client
+	var err error
+
+	if apiVersion, ok := os.LookupEnv("DOCKER_API_VERSION"); ok {
+		cli, err = client.NewClientWithOpts(
+			client.FromEnv,
+			client.WithVersion(apiVersion),
+		)
+	} else {
+		cli, err = client.NewClientWithOpts(
+			client.FromEnv,
+		)
+	}
+	if err != nil {
+		return "", err
+	}
+	defer func(cli *client.Client) {
+		err := cli.Close()
+		if err != nil {
+			t.Logf("failed to close docker client")
+		}
+	}(cli)
+
+	var sparkContainerID string
+	if _, ok := os.LookupEnv("SPARK_CONTAINER_ID"); ok {
+		sparkContainerID = os.Getenv("SPARK_CONTAINER_ID")
+	} else {
+		filter := filters.NewArgs(filters.Arg("name", sparkContainer))
+		containers, err := cli.ContainerList(t.Context(), container.ListOptions{
+			Filters: filter,
+		})
+		if err != nil {
+			return "", err
+		}
+		if len(containers) != 1 {
+			return "", fmt.Errorf("unable to find container: %s", sparkContainer)
+		}
+		sparkContainerID = containers[0].ID
+	}
+
+	response, err := cli.ContainerExecCreate(t.Context(), sparkContainerID, container.ExecOptions{
+		Cmd:          append([]string{"python", scriptPath}, args...),
+		AttachStdout: true,
+		AttachStderr: true,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	attachResp, err := cli.ContainerExecAttach(t.Context(), response.ID, container.ExecAttachOptions{})
+	if err != nil {
+		return "", err
+	}
+	defer attachResp.Close()
+
+	output, err := io.ReadAll(attachResp.Reader)
+	if err != nil {
+		return "", err
+	}
+	fmt.Printf("%s\n", string(output))
+
+	inspect, err := cli.ContainerExecInspect(t.Context(), response.ID)
+	if err != nil {
+		return "", err
+	}
+	if inspect.ExitCode != 0 {
+		return "", fmt.Errorf("failed to execute script with exit code: %d", inspect.ExitCode)
+	}
+
+	return string(output), nil
+}
diff --git a/internal/recipe/run_spark_count_sql.py b/internal/recipe/run_spark_count_sql.py
deleted file mode 100644
index 9c472ff..0000000
--- a/internal/recipe/run_spark_count_sql.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyspark.sql import SparkSession
-
-spark = SparkSession.builder.getOrCreate()
-
-spark.sql("SELECT COUNT(*) FROM default.test_partitioned_by_days").show()
diff --git a/internal/recipe/validation.py b/internal/recipe/validation.py
new file mode 100644
index 0000000..1ac9421
--- /dev/null
+++ b/internal/recipe/validation.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.getOrCreate()
+
+
+def testSetProperties():
+    spark.sql("SHOW TBLPROPERTIES default.go_test_set_properties").show(truncate=False)
+
+
+def testAddedFile():
+    spark.sql("SELECT COUNT(*) FROM default.test_partitioned_by_days").show(truncate=False)
+
+
+def testReadDifferentDataTypes():
+    spark.sql("DESCRIBE TABLE EXTENDED default.go_test_different_data_types").show(truncate=False)
+    spark.sql("SELECT * FROM default.go_test_different_data_types").show(truncate=False)
+
+
+def testReadSpecUpdate():
+    spark.sql("DESCRIBE TABLE EXTENDED default.go_test_update_spec").show(truncate=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--test", type=str, required=True, help="Name of the test to run")
+    args = parser.parse_args()
+
+    if args.test == "TestSetProperties":
+        testSetProperties()
+
+    if args.test == "TestAddedFile":
+        testAddedFile()
+
+    if args.test == "TestReadDifferentDataTypes":
+        testReadDifferentDataTypes()
+
+    if args.test == "TestReadSpecUpdate":
+        testReadSpecUpdate()
diff --git a/table/transaction_test.go b/table/transaction_test.go
index 4f68bed..e968eaf 100644
--- a/table/transaction_test.go
+++ b/table/transaction_test.go
@@ -21,7 +21,6 @@ package table_test
 
 import (
 	"context"
-	"io"
 	"strings"
 	"testing"
 	"time"
@@ -54,9 +53,6 @@ func (s *SparkIntegrationTestSuite) SetupSuite() {
 	var err error
 	s.stack, err = recipe.Start(s.T())
 	s.Require().NoError(err)
-	if s.stack == nil {
-		s.T().Skip("skipping test, AWS_S3_ENDPOINT is set")
-	}
 }
 
 func (s *SparkIntegrationTestSuite) SetupTest() {
@@ -73,6 +69,51 @@ func (s *SparkIntegrationTestSuite) SetupTest() {
 	s.cat = cat
 }
 
+func (s *SparkIntegrationTestSuite) TestSetProperties() {
+	icebergSchema := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.Bool},
+		iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Int32},
+	)
+
+	tbl, err := s.cat.CreateTable(s.ctx, catalog.ToIdentifier("default", "go_test_set_properties"), icebergSchema)
+	s.Require().NoError(err)
+
+	tx := tbl.NewTransaction()
+
+	err = tx.SetProperties(iceberg.Properties{
+		table.ParquetCompressionKey:      "snappy",
+		table.ManifestMergeEnabledKey:    "true",
+		table.ManifestMinMergeCountKey:   "1",
+		table.ManifestTargetSizeBytesKey: "1",
+	})
+	s.Require().NoError(err)
+
+	_, err = tx.Commit(s.ctx)
+	s.Require().NoError(err)
+
+	expectedOutput := `
++----------------------------------+---------------+
+|key                               |value          |
++----------------------------------+---------------+
+|commit.manifest-merge.enabled     |true           |
+|commit.manifest.min-count-to-merge|1              |
+|commit.manifest.target-size-bytes |1              |
+|current-snapshot-id               |none           |
+|format                            |iceberg/parquet|
+|format-version                    |2              |
+|write.parquet.compression-codec   |snappy         |
++----------------------------------+---------------+
+`
+
+	output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestSetProperties")
+	s.Require().NoError(err)
+	s.Require().True(
+		strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+		"result does not contain expected output: %s", expectedOutput,
+	)
+}
+
 func (s *SparkIntegrationTestSuite) TestAddFile() {
 	const filename = "s3://warehouse/default/test_partitioned_by_days/data/ts_day=2023-03-13/supertest.parquet"
 
@@ -114,21 +155,210 @@ func (s *SparkIntegrationTestSuite) TestAddFile() {
 	tbl, err = tx.Commit(s.ctx)
 	s.Require().NoError(err)
 
-	spark, err := s.stack.ServiceContainer(s.T().Context(), "spark-iceberg")
+	expectedOutput := `
++--------+
+|count(1)|
++--------+
+|13      |
++--------+
+`
+
+	output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestAddedFile")
 	s.Require().NoError(err)
+	s.Require().True(
+		strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+		"result does not contain expected output: %s", expectedOutput,
+	)
+}
+
+func (s *SparkIntegrationTestSuite) TestDifferentDataTypes() {
+	icebergSchema := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "bool", Type: iceberg.PrimitiveTypes.Bool},
+		iceberg.NestedField{ID: 2, Name: "string", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 3, Name: "string_long", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 4, Name: "int", Type: iceberg.PrimitiveTypes.Int32},
+		iceberg.NestedField{ID: 5, Name: "long", Type: iceberg.PrimitiveTypes.Int64},
+		iceberg.NestedField{ID: 6, Name: "float", Type: iceberg.PrimitiveTypes.Float32},
+		iceberg.NestedField{ID: 7, Name: "double", Type: iceberg.PrimitiveTypes.Float64},
+		iceberg.NestedField{ID: 8, Name: "timestamp", Type: iceberg.PrimitiveTypes.Timestamp},
+		iceberg.NestedField{ID: 9, Name: "timestamptz", Type: iceberg.PrimitiveTypes.TimestampTz},
+		iceberg.NestedField{ID: 10, Name: "date", Type: iceberg.PrimitiveTypes.Date},
+		iceberg.NestedField{ID: 11, Name: "uuid", Type: iceberg.PrimitiveTypes.UUID},
+		iceberg.NestedField{ID: 12, Name: "binary", Type: iceberg.PrimitiveTypes.Binary},
+		iceberg.NestedField{ID: 13, Name: "fixed", Type: iceberg.FixedTypeOf(16)},
+		iceberg.NestedField{ID: 14, Name: "small_dec", Type: iceberg.DecimalTypeOf(8, 2)},
+		iceberg.NestedField{ID: 15, Name: "med_dec", Type: iceberg.DecimalTypeOf(16, 2)},
+		iceberg.NestedField{ID: 16, Name: "large_dec", Type: iceberg.DecimalTypeOf(24, 2)},
+		iceberg.NestedField{ID: 17, Name: "list", Type: &iceberg.ListType{
+			ElementID: 18,
+			Element:   iceberg.PrimitiveTypes.Int32},
+		},
+	)
 
-	_, stdout, err := spark.Exec(s.ctx, []string{"ipython", "./run_spark_count_sql.py"})
+	arrowSchema, err := table.SchemaToArrowSchema(icebergSchema, nil, true, false)
 	s.Require().NoError(err)
 
-	output, err := io.ReadAll(stdout)
+	arrTable, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{
+		`[
+			{
+				"bool": false,
+				"string": "a",
+				"string_long": "` + strings.Repeat("a", 22) + `",
+				"int": 1,
+				"long": 1,
+				"float": 0.0,
+				"double": 0.0,
+				"timestamp": "2023-01-01T19:25:00.000000+08:00",
+				"timestamptz": "2023-01-01T19:25:00.000000Z",
+				"date": "2023-01-01",
+				"uuid": "00000000-0000-0000-0000-000000000000",
+				"binary": "AQ==",
+				"fixed": "AAAAAAAAAAAAAAAAAAAAAA==",
+				"small_dec": "123456.78",
+				"med_dec": "12345678901234.56",
+				"large_dec": "1234567890123456789012.34",
+				"list": [1, 2, 3]
+			},
+			{
+				"bool": null,
+				"string": null,
+				"string_long": null,
+				"int": null,
+				"long": null,
+				"float": null,
+				"double": null,
+				"timestamp": null,
+				"timestamptz": null,
+				"date": null,
+				"uuid": null,
+				"binary": null,
+				"fixed": null,
+				"small_dec": null,
+				"med_dec": null,
+				"large_dec": null,
+				"list": null
+			},
+			{
+				"bool": true,
+				"string": "z",
+				"string_long": "` + strings.Repeat("z", 22) + `",
+				"int": 9,
+				"long": 9,
+				"float": 0.9,
+				"double": 0.9,
+				"timestamp": "2023-03-01T19:25:00.000000+08:00",
+				"timestamptz": "2023-03-01T19:25:00.000000Z",
+				"date": "2023-03-01",
+				"uuid": "11111111-1111-1111-1111-111111111111",
+				"binary": "Eg==",
+				"fixed": "EREREREREREREREREREREQ==",
+				"small_dec": "876543.21",
+				"med_dec": "65432109876543.21",
+				"large_dec": "4321098765432109876543.21",
+				"list": [-1, -2, -3]
+			}
+		 ]`,
+	})
 	s.Require().NoError(err)
-	strings.HasSuffix(string(output), `
-+--------+
-|count(1)|
-+--------+
-|      13|
-+--------+
-`)
+	defer arrTable.Release()
+
+	tbl, err := s.cat.CreateTable(s.ctx, catalog.ToIdentifier("default", "go_test_different_data_types"), icebergSchema)
+	s.Require().NoError(err)
+
+	tx := tbl.NewTransaction()
+	err = tx.AppendTable(s.ctx, arrTable, 1, nil)
+	s.Require().NoError(err)
+
+	_, err = tx.Commit(s.ctx)
+	s.Require().NoError(err)
+
+	expectedSchema := `
++------------------+-------------+-------+
+|col_name          |data_type    |comment|
++------------------+-------------+-------+
+|bool              |boolean      |NULL   |
+|string            |string       |NULL   |
+|string_long       |string       |NULL   |
+|int               |int          |NULL   |
+|long              |bigint       |NULL   |
+|float             |float        |NULL   |
+|double            |double       |NULL   |
+|timestamp         |timestamp_ntz|NULL   |
+|timestamptz       |timestamp    |NULL   |
+|date              |date         |NULL   |
+|uuid              |string       |NULL   |
+|binary            |binary       |NULL   |
+|fixed             |binary       |NULL   |
+|small_dec         |decimal(8,2) |NULL   |
+|med_dec           |decimal(16,2)|NULL   |
+|large_dec         |decimal(24,2)|NULL   |
+|list              |array<int>   |NULL   |
+`
+
+	expectedOutput := `
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+|bool |string|string_long           |int |long|float|double|timestamp          |timestamptz        |date      |uuid                                |binary|fixed                                            |small_dec|med_dec          |large_dec                |list        |
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+|false|a     |aaaaaaaaaaaaaaaaaaaaaa|1   |1   |0.0  |0.0   |2023-01-01 11:25:00|2023-01-01 19:25:00|2023-01-01|00000000-0000-0000-0000-000000000000|[01]  |[00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]|123456.78|12345678901234.56|1234567890123456789012.34|[1, 2, 3]   |
+|NULL |NULL  |NULL                  |NULL|NULL|NULL |NULL  |NULL               |NULL               |NULL      |NULL                                |NULL  |NULL                                             |NULL     |NULL             |NULL                     |NULL        |
+|true |z     |zzzzzzzzzzzzzzzzzzzzzz|9   |9   |0.9  |0.9   |2023-03-01 11:25:00|2023-03-01 19:25:00|2023-03-01|11111111-1111-1111-1111-111111111111|[12]  |[11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11]|876543.21|65432109876543.21|4321098765432109876543.21|[-1, -2, -3]|
++-----+------+----------------------+----+----+-----+------+-------------------+-------------------+----------+------------------------------------+------+-------------------------------------------------+---------+-----------------+-------------------------+------------+
+`
+
+	output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestReadDifferentDataTypes")
+	s.Require().NoError(err)
+	s.Require().True(
+		strings.Contains(strings.TrimSpace(output), strings.TrimSpace(expectedSchema)),
+		"result does not contain expected output: %s", expectedOutput,
+	)
+	s.Require().True(
+		strings.HasSuffix(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)),
+		"result does not contain expected output: %s", expectedOutput,
+	)
+}
+
+func (s *SparkIntegrationTestSuite) TestUpdateSpec() {
+	icebergSchema := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.Bool},
+		iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Int32},
+	)
+
+	partitionSpec := iceberg.NewPartitionSpec(
+		iceberg.PartitionField{SourceID: 2, FieldID: 1000, Transform: iceberg.TruncateTransform{Width: 5}, Name: "bar_truncate"},
+	)
+
+	tbl, err := s.cat.CreateTable(
+		s.ctx,
+		catalog.ToIdentifier("default", "go_test_update_spec"),
+		icebergSchema,
+		catalog.WithPartitionSpec(&partitionSpec),
+	)
+	s.Require().NoError(err)
+
+	tx := tbl.NewTransaction()
+	err = tx.UpdateSpec(false).
+		AddField("baz", iceberg.BucketTransform{NumBuckets: 3}, "").
+		Commit()
+	s.Require().NoError(err)
+	_, err = tx.Commit(s.ctx)
+
+	partitionExpectedOutput := `
+|# Partitioning              |                                            |       |
+|Part 0                      |truncate(5, bar)                            |       |
+|Part 1                      |bucket(3, baz)                              |       |
+|                            |                                            |       |
+`
+	metadataPartition := `
+|_partition                  |struct<bar_truncate:string,baz_bucket_3:int>|       |
+`
+
+	output, err := recipe.ExecuteSpark(s.T(), "./validation.py", "--test", "TestReadSpecUpdate")
+	s.Require().NoError(err)
+	s.Require().True(
+		strings.Contains(strings.TrimSpace(output), strings.TrimSpace(partitionExpectedOutput)),
+		"result does not contain expected output: %s", metadataPartition,
+	)
 }
 
 func TestSparkIntegration(t *testing.T) {