(texera) 01/01: test(workflow-operator): cover the vectorizer branches of Sklearn code generation (#7577)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-7577-d61214a689be3b83eaa27499efe7daa13de71671
in repository https://gitbox.apache.org/repos/asf/texera.git

commit e46564f178e86feab51255d785d67fe3d8a49df0
Author: Eugene Gu <eu...@outlook.com>
AuthorDate: Thu Aug 13 03:59:24 2026 +0000

    test(workflow-operator): cover the vectorizer branches of Sklearn code generation (#7577)
    
    ### What changes were proposed in this PR?
    
    This PR adds unit test coverage for the vectorizer branches of the
    Python code generation in the two shared Sklearn base descriptors:
    
    - `SklearnTrainingOpDesc` (base of the 26 Sklearn training operators)
    - `SklearnClassifierOpDesc` (base of the 25 Sklearn classifier
    operators)
    
    Both templates branch on the `countVectorizer` and `tfidfTransformer`
    properties to select the text column and prepend `CountVectorizer()` /
    `TfidfTransformer()` stages to the generated `make_pipeline` call, but
    no test in the repository generated code with either flag set: every
    existing `generatePythonCode()` assertion runs with both flags
    default-false, and the specs that do set `countVectorizer = true` are
    Jackson round-trip tests that never invoke code generation.
    
    Two new specs exercise each base through a representative concrete
    subclass (`SklearnTrainingKNNOpDesc` / `SklearnKNNOpDesc`), matching how
    the operators use the bases:
    
    - `SklearnTrainingOpDescCodegenSpec` (4 tests)
    - `SklearnClassifierOpDescCodegenSpec` (4 tests)
    
    Each spec covers all four flag combinations the templates distinguish,
    with positive and negative assertions: the both-false baseline
    (whole-feature path, no vectorizer stages), `countVectorizer` alone
    (text-column selection plus `CountVectorizer()` stage), both flags
    (stage order asserted via the full `make_pipeline(CountVectorizer(),
    TfidfTransformer(), ...)` call), and `tfidfTransformer` alone (a
    reachable codegen branch even though the UI hides the field when
    `countVectorizer` is off). Attribute names are `EncodableString`s, so
    the expected values are built with the production
    `PythonTemplateBuilder.wrapWithPythonDecoderExpr`, pinning the real
    base64 decode expressions in the generated code.
    
    No production code is changed.
    
    ### Any related issues, documentation, discussions?
    
    Closes #7574
    
    ### How was this PR tested?
    
    This PR is itself test-only. The new specs were run with:
    
    ```
    sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDescCodegenSpec org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDescCodegenSpec"
    ```
    
    All 8 tests pass. The suite was additionally mutation-checked: six
    manual template mutations (swapping the
    `CountVectorizer`/`TfidfTransformer` stage order, gating the text-column
    selection on the wrong flag, and ignoring the `tfidfTransformer` flag,
    in each base) each caused test failures, and the sources were restored
    afterwards. `scalafmtCheck` passes.
    
    ### Was this PR authored or co-authored using generative AI tooling?
    
    Co-authored by: Claude Code (Claude Fable 5)
---
 .../SklearnClassifierOpDescCodegenSpec.scala       | 111 +++++++++++++++++++++
 .../SklearnTrainingOpDescCodegenSpec.scala         | 108 ++++++++++++++++++++
 2 files changed, 219 insertions(+)

diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala
new file mode 100644
index 0000000000..9e732002eb
--- /dev/null
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.operator.sklearn
+
+import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+/**
+  * Covers the vectorizer branches of the shared train-and-evaluate codegen
+  * template in SklearnClassifierOpDesc. The abstract base is exercised through
+  * a representative concrete subclass (SklearnKNNOpDesc), matching how the 25
+  * classifier operators use it.
+  *
+  * The template distinguishes all four flag combinations: countVectorizer
+  * gates the text-column selection and the CountVectorizer() stage, while
+  * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI
+  * hides it when countVectorizer is off, but codegen still honors it).
+  */
+class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers {
+
+  // UI-provided attribute names are EncodableStrings: the encoded template
+  // renders them as a runtime base64-decode expression, not a Python literal.
+  private def decodeExpr(value: String): String =
+    PythonTemplateBuilder.wrapWithPythonDecoderExpr(
+      Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8))
+    )
+
+  // Collapse space runs so assertions on the make_pipeline call are not
+  // coupled to the exact spacing the empty branches leave behind.
+  private def normalized(code: String): String = code.replaceAll(" +", " ")
+
+  private def descriptor(
+      countVectorizer: Boolean = false,
+      tfidfTransformer: Boolean = false
+  ): SklearnKNNOpDesc = {
+    val d = new SklearnKNNOpDesc
+    d.target = "label"
+    d.text = "docs"
+    d.countVectorizer = countVectorizer
+    d.tfidfTransformer = tfidfTransformer
+    d
+  }
+
+  "SklearnClassifierOpDesc.generatePythonCode" should
+    "generate a plain feature pipeline when both vectorizer flags are off" in {
+    val code = descriptor().generatePythonCode()
+    code should include("from sklearn.neighbors import KNeighborsClassifier")
+    code should include(s"Y = table[${decodeExpr("label")}]")
+    code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)")
+    // Feature-column path: X is kept whole, the text attribute is never read.
+    code should include("X = X\n")
+    code should not include decodeExpr("docs")
+    normalized(code) should include(
+      "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)"
+    )
+    code should include("predictions = self.model.predict(X)")
+    code should not include "CountVectorizer()"
+    code should not include "TfidfTransformer()"
+  }
+
+  it should "select the text column and prepend CountVectorizer when countVectorizer is on" in {
+    val code = descriptor(countVectorizer = true).generatePythonCode()
+    code should include(s"X = X[${decodeExpr("docs")}]")
+    code should not include "X = X\n"
+    normalized(code) should include(
+      "self.model = make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+    code should not include "TfidfTransformer()"
+  }
+
+  it should "chain CountVectorizer before TfidfTransformer when both flags are on" in {
+    val code =
+      descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode()
+    code should include(s"X = X[${decodeExpr("docs")}]")
+    normalized(code) should include(
+      "self.model = make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+  }
+
+  it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in {
+    val code = descriptor(tfidfTransformer = true).generatePythonCode()
+    // Without countVectorizer there is no text-column selection.
+    code should include("X = X\n")
+    code should not include decodeExpr("docs")
+    normalized(code) should include(
+      "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+    code should not include "CountVectorizer()"
+  }
+}
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala
new file mode 100644
index 0000000000..0c00ca271b
--- /dev/null
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.operator.sklearn.training
+
+import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+/**
+  * Covers the vectorizer branches of the shared training codegen template in
+  * SklearnTrainingOpDesc. The template is exercised through a representative
+  * concrete subclass (SklearnTrainingKNNOpDesc) so the generated pipeline ends
+  * in a real estimator, matching how the 26 training operators use the base.
+  *
+  * The template distinguishes all four flag combinations: countVectorizer
+  * gates the text-column selection and the CountVectorizer() stage, while
+  * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI
+  * hides it when countVectorizer is off, but codegen still honors it).
+  */
+class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers {
+
+  // UI-provided attribute names are EncodableStrings: the encoded template
+  // renders them as a runtime base64-decode expression, not a Python literal.
+  private def decodeExpr(value: String): String =
+    PythonTemplateBuilder.wrapWithPythonDecoderExpr(
+      Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8))
+    )
+
+  // Collapse space runs so assertions on the make_pipeline call are not
+  // coupled to the exact spacing the empty branches leave behind.
+  private def normalized(code: String): String = code.replaceAll(" +", " ")
+
+  private def descriptor(
+      countVectorizer: Boolean = false,
+      tfidfTransformer: Boolean = false
+  ): SklearnTrainingKNNOpDesc = {
+    val d = new SklearnTrainingKNNOpDesc
+    d.target = "label"
+    d.text = "docs"
+    d.countVectorizer = countVectorizer
+    d.tfidfTransformer = tfidfTransformer
+    d
+  }
+
+  "SklearnTrainingOpDesc.generatePythonCode" should
+    "generate a plain feature pipeline when both vectorizer flags are off" in {
+    val code = descriptor().generatePythonCode()
+    code should include("from sklearn.neighbors import KNeighborsClassifier")
+    code should include(s"Y = table[${decodeExpr("label")}]")
+    code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)")
+    // Feature-column path: X is kept whole, the text attribute is never read.
+    code should include("X = X\n")
+    code should not include decodeExpr("docs")
+    normalized(code) should include("make_pipeline( KNeighborsClassifier()).fit(X, Y)")
+    code should not include "CountVectorizer()"
+    code should not include "TfidfTransformer()"
+  }
+
+  it should "select the text column and prepend CountVectorizer when countVectorizer is on" in {
+    val code = descriptor(countVectorizer = true).generatePythonCode()
+    code should include(s"X = X[${decodeExpr("docs")}]")
+    code should not include "X = X\n"
+    normalized(code) should include(
+      "make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+    code should not include "TfidfTransformer()"
+  }
+
+  it should "chain CountVectorizer before TfidfTransformer when both flags are on" in {
+    val code =
+      descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode()
+    code should include(s"X = X[${decodeExpr("docs")}]")
+    normalized(code) should include(
+      "make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+  }
+
+  it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in {
+    val code = descriptor(tfidfTransformer = true).generatePythonCode()
+    // Without countVectorizer there is no text-column selection.
+    code should include("X = X\n")
+    code should not include decodeExpr("docs")
+    normalized(code) should include(
+      "make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"
+    )
+    code should not include "CountVectorizer()"
+  }
+}