You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "hudi-agent (via GitHub)" <gi...@apache.org> on 2026/05/01 00:28:51 UTC
Re: [PR] feat(integ-test): add Testcontainers-based E2E tests for Vector/Blob/… [hudi]
hudi-agent commented on code in PR #18535:
URL: https://github.com/apache/hudi/pull/18535#discussion_r3171538020
##########
hudi-spark-datasource/hudi-spark4-common/src/main/java/org/apache/hudi/io/storage/Spark4VariantProjectedRow.java:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.io.storage;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters;
+import org.apache.spark.sql.catalyst.util.ArrayData;
+import org.apache.spark.sql.catalyst.util.MapData;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.unsafe.types.CalendarInterval;
+import org.apache.spark.unsafe.types.UTF8String;
+import org.apache.spark.unsafe.types.VariantVal;
+
+import java.util.List;
+import java.util.function.BiConsumer;
+
+/**
+ * Spark 4 implementation of {@link VariantProjectedRow}. Pass-through
+ * {@link InternalRow} that delegates every accessor to the wrapped input row
+ * except at variant ordinals, where it returns a pre-allocated
+ * {@code (metadata, value)} {@link GenericInternalRow} that the extractor
+ * populates on demand.
+ *
+ * <p>Single-row, single-threaded; {@link #wrap(InternalRow)} mutates state.
+ * {@link #copy()} / {@link #update(int, Object)} / {@link #setNullAt(int)}
+ * throw - lance-spark consumes each row synchronously inside
+ * {@code LanceArrowWriter.write(InternalRow)}, so no copy is ever needed.
+ */
+public abstract class Spark4VariantProjectedRow extends InternalRow implements VariantProjectedRow {
+
+ private final int numFields;
+ private final GenericInternalRow[] variantStructByOrdinal;
+ private final List<BiConsumer<SpecializedGetters, Integer>> extractorByOrdinal;
+ protected InternalRow input;
+
+ public Spark4VariantProjectedRow(int numFields,
+ GenericInternalRow[] variantStructByOrdinal,
+ List<BiConsumer<SpecializedGetters, Integer>> extractorByOrdinal) {
+ this.numFields = numFields;
+ this.variantStructByOrdinal = variantStructByOrdinal;
+ this.extractorByOrdinal = extractorByOrdinal;
+ }
+
+ @Override
+ public InternalRow wrap(InternalRow input) {
+ this.input = input;
+ return this;
+ }
+
+ @Override
+ public int numFields() {
+ return numFields;
+ }
+
+ @Override
+ public boolean isNullAt(int ordinal) {
+ return input.isNullAt(ordinal);
+ }
+
+ @Override
+ public InternalRow getStruct(int ordinal, int n) {
+ GenericInternalRow variantStruct = variantStructByOrdinal[ordinal];
+ if (variantStruct != null) {
+ // Populate metadata (slot 0) and value (slot 1) via the captured lambdas.
+ extractorByOrdinal.get(ordinal).accept(input, ordinal);
+ return variantStruct;
+ }
+ return input.getStruct(ordinal, n);
+ }
+
+ @Override
+ public boolean getBoolean(int ordinal) {
+ return input.getBoolean(ordinal);
+ }
+
+ @Override
+ public byte getByte(int ordinal) {
+ return input.getByte(ordinal);
+ }
+
+ @Override
+ public short getShort(int ordinal) {
+ return input.getShort(ordinal);
+ }
+
+ @Override
+ public int getInt(int ordinal) {
+ return input.getInt(ordinal);
+ }
+
+ @Override
+ public long getLong(int ordinal) {
+ return input.getLong(ordinal);
+ }
+
+ @Override
+ public float getFloat(int ordinal) {
+ return input.getFloat(ordinal);
+ }
+
+ @Override
+ public double getDouble(int ordinal) {
+ return input.getDouble(ordinal);
+ }
+
+ @Override
+ public Decimal getDecimal(int ordinal, int precision, int scale) {
+ return input.getDecimal(ordinal, precision, scale);
+ }
+
+ @Override
+ public UTF8String getUTF8String(int ordinal) {
+ return input.getUTF8String(ordinal);
+ }
+
+ @Override
+ public byte[] getBinary(int ordinal) {
+ return input.getBinary(ordinal);
+ }
+
+ @Override
+ public CalendarInterval getInterval(int ordinal) {
+ return input.getInterval(ordinal);
+ }
+
+ @Override
+ public VariantVal getVariant(int ordinal) {
+ return input.getVariant(ordinal);
+ }
+
+ @Override
+ public ArrayData getArray(int ordinal) {
+ return input.getArray(ordinal);
+ }
+
+ @Override
Review Comment:
🤖 nit: could you give these single-letter parameters meaningful names (e.g. `ordinal`, `value`) to match the rest of the file's style? The other accessors here all use `ordinal`.
<sub><i>- AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub>
##########
hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/command/CommandResult.java:
##########
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ2.testcontainers.command;
+
+import lombok.AllArgsConstructor;
+import org.testcontainers.containers.Container;
+
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+/**
+ * A dedicated class to hold the result of a command execution and provide
+ * fluent assertion methods for cleaner tests.
+ */
+@AllArgsConstructor
+public class CommandResult {
+
+ // Spark 4.0 (Scala 2.13) spark-shell runs under a dumb terminal when stdin
+ // is piped from a file, which flushes the next `scala> ` prompt onto the
+ // same line as preceding async println output (e.g. `scala> MARKER`).
+ // Stripping one-or-more leading `scala>\s+` prefixes normalizes those lines
+ // back to the bare sentinel while leaving Spark 3.5 output (sentinel already
+ // on its own line) unchanged.
+ private static final Pattern REPL_PROMPT_PREFIX = Pattern.compile("^(scala>\\s+)+");
+
+ private final String stdout;
+ private final String stderr;
+ private final int exitCode;
+
+ public CommandResult(Container.ExecResult execResult) {
+ this.stdout = execResult.getStdout();
+ this.stderr = execResult.getStderr();
+ this.exitCode = execResult.getExitCode();
+ }
+
+ /**
+ * Asserts that the command's exit code is 0 (success).
+ *
+ * @return The same {@link CommandResult} instance for chaining assertions.
+ * @throws AssertionError if the exit code is not 0.
+ */
+ public CommandResult expectToSucceed() {
+ assertEquals(0, exitCode,
+ String.format("Command failed with exit code %d. Stderr: %s", exitCode, stderr));
+ return this;
+ }
+
+ /**
+ * Asserts that the command's exit code is not 0 (failure).
+ * More specifically, it asserts the exit code is 1.
+ *
+ * @return The same {@link CommandResult} instance for chaining assertions.
+ * @throws AssertionError if the exit code is 0.
+ */
+ public CommandResult expectToFail() {
+ assertNotEquals(0, exitCode,
+ String.format("Command succeeded with exit code %d. Stderr: %s", exitCode, stderr));
+ return this;
+ }
+
+ /**
+ * Asserts that the command's exit code is zero.
Review Comment:
🤖 nit: the Javadoc says "exit code is zero" but the method takes an `expectedCode` parameter and asserts against that. Could you update the doc to something like "Asserts that the command's exit code equals `expectedCode`."?
<sub><i>- AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub>
##########
hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java:
##########
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ2.testcontainers;
+
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * Central configuration for the integ2 Testcontainers harness. Every constant that
+ * describes "how this harness expects the compose environment to look" belongs here:
+ * container service names, container-side paths, network endpoints, timeouts, and
+ * the keys / defaults of system properties that tune the harness.
+ *
+ * <p>Per-test fixture data (table paths, per-test {@code .commands} scripts) does
+ * not belong here, those stay in the relevant test class, though they should build
+ * their common prefix from {@link Paths#DEMO_DIR} / {@link Paths#WS_ROOT} rather
+ * than inlining the path literal.
+ */
+public final class TestcontainersConfig {
+
+ private TestcontainersConfig() {
+ }
+
+ /** Docker Compose service names. Must match the compose YAML verbatim. */
+ public static final class Containers {
+ public static final String NAMENODE = "namenode";
+ public static final String DATANODE_1 = "datanode1";
+ public static final String HISTORY_SERVER = "historyserver";
+ public static final String HIVE_METASTORE_PGSQL = "hive-metastore-postgresql";
+ public static final String HIVE_METASTORE = "hivemetastore";
+ public static final String HIVESERVER = "hiveserver";
+ public static final String ZOOKEEPER = "zookeeper";
+ // TODO: The value intentionally matches SPARK_MASTER, preserved from the
+ // pre-refactor source to avoid changing behavior. If the compose YAML has a
+ // distinct sparkworker1 service, correct this in a follow-up commit.
+ public static final String SPARK_WORKER_1 = "sparkmaster";
+ public static final String KAFKA = "kafka";
+ public static final String SPARK_MASTER = "sparkmaster";
Review Comment:
🤖 nit: a constant named `SPARK_WORKER_1` that resolves to `"sparkmaster"` is actively misleading — a future developer reading `provider.getContainer(Containers.SPARK_WORKER_1)` will assume it targets a worker. Could you either alias it to `SPARK_MASTER` (`public static final String SPARK_WORKER_1 = SPARK_MASTER;`) or, better, drop it entirely until the compose YAML has a real `sparkworker1` service?
<sub><i>- AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org