You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2025/06/05 03:44:35 UTC
(spark) branch master updated (28f05886a078 -> 4c69d0df96cb)
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 28f05886a078 [MINOR] Ensure structured_logging_style runs out of correct directory
add 4c69d0df96cb [SPARK-52283][SQL] Declarative Pipelines `DataflowGraph` creation and resolution
No new revisions were added by this update.
Summary of changes:
.../src/main/resources/error/error-conditions.json | 71 +++
sql/pipelines/pom.xml | 76 +++-
.../spark/sql/pipelines/AnalysisWarning.scala | 37 +-
.../org/apache/spark/sql/pipelines/Language.scala | 12 +-
.../graph/CoreDataflowNodeProcessor.scala | 222 +++++++++
.../spark/sql/pipelines/graph/DataflowGraph.scala | 239 ++++++++++
.../pipelines/graph/DataflowGraphTransformer.scala | 378 ++++++++++++++++
.../apache/spark/sql/pipelines/graph/Flow.scala | 196 ++++++++
.../spark/sql/pipelines/graph/FlowAnalysis.scala | 313 +++++++++++++
.../sql/pipelines/graph/FlowAnalysisContext.scala | 72 +++
.../pipelines/graph/GraphElementTypeUtils.scala} | 25 +-
.../spark/sql/pipelines/graph/GraphErrors.scala | 131 ++++++
.../pipelines/graph/GraphIdentifierManager.scala | 333 ++++++++++++++
.../sql/pipelines/graph/GraphOperations.scala | 177 ++++++++
.../pipelines/graph/GraphRegistrationContext.scala | 211 +++++++++
.../sql/pipelines/graph/GraphValidations.scala | 267 +++++++++++
.../sql/pipelines/graph/PipelinesErrors.scala | 93 ++++
.../pipelines/graph/PipelinesTableProperties.scala | 112 +++++
.../spark/sql/pipelines/graph/QueryOrigin.scala | 148 ++++++
.../spark/sql/pipelines/graph/ViewHelpers.scala} | 28 +-
.../spark/sql/pipelines/graph/elements.scala | 270 +++++++++++
.../spark/sql/pipelines/util/InputReadInfo.scala | 48 ++
.../sql/pipelines/util/SchemaInferenceUtils.scala | 172 +++++++
.../sql/pipelines/util/SchemaMergingUtils.scala} | 9 +-
.../graph/ConnectInvalidPipelineSuite.scala | 469 +++++++++++++++++++
.../graph/ConnectValidPipelineSuite.scala | 500 +++++++++++++++++++++
.../pipelines/util/SchemaInferenceUtilsSuite.scala | 273 +++++++++++
.../spark/sql/pipelines/utils/PipelineTest.scala | 461 +++++++++++++++++++
.../sql/pipelines/utils/SparkErrorTestMixin.scala | 105 +++++
.../utils/TestGraphRegistrationContext.scala | 226 ++++++++++
30 files changed, 5612 insertions(+), 62 deletions(-)
copy common/kvstore/src/test/java/org/apache/spark/util/kvstore/IntKeyType.java => sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/AnalysisWarning.scala (61%)
copy common/utils/src/main/java/org/apache/spark/memory/MemoryMode.java => sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/Language.scala (83%)
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/CoreDataflowNodeProcessor.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraph.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/FlowAnalysis.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/FlowAnalysisContext.scala
copy sql/{hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/TableTypeMappingFactory.java => pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphElementTypeUtils.scala} (57%)
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphErrors.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphIdentifierManager.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphOperations.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphRegistrationContext.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/GraphValidations.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelinesErrors.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelinesTableProperties.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/QueryOrigin.scala
copy sql/{core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTableCommand.scala => pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/ViewHelpers.scala} (57%)
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/elements.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/InputReadInfo.scala
create mode 100644 sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtils.scala
copy sql/{catalyst/src/test/scala/org/apache/spark/sql/catalyst/OrderedFiltersSuite.scala => pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaMergingUtils.scala} (76%)
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectInvalidPipelineSuite.scala
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/PipelineTest.scala
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/SparkErrorTestMixin.scala
create mode 100644 sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/TestGraphRegistrationContext.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org