You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2022/03/15 20:31:32 UTC
[spark] branch master updated (21db916 -> 4e31000)
This is an automated email from the ASF dual-hosted git repository.
liyuanjian pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git.
from 21db916 [SPARK-38484][PYTHON] Move usage logging instrumentation util functions from pandas module to pyspark.util module
add 4e31000 [SPARK-38204][SS] Use StatefulOpClusteredDistribution for stateful operators with respecting backward compatibility
No new revisions were added by this update.
Summary of changes:
docs/ss-migration-guide.md | 4 +
.../org/apache/spark/sql/internal/SQLConf.scala | 17 +
.../spark/sql/execution/aggregate/AggUtils.scala | 64 ++-
.../execution/aggregate/BaseAggregateExec.scala | 23 +-
.../execution/aggregate/HashAggregateExec.scala | 2 +
.../execution/aggregate/MergingSessionsExec.scala | 15 +-
.../aggregate/ObjectHashAggregateExec.scala | 2 +
.../execution/aggregate/SortAggregateExec.scala | 2 +
.../execution/aggregate/UpdatingSessionsExec.scala | 18 +-
.../streaming/FlatMapGroupsWithStateExec.scala | 13 +-
.../execution/streaming/IncrementalExecution.scala | 17 +
.../spark/sql/execution/streaming/OffsetSeq.scala | 5 +-
.../streaming/StatefulOperatorPartitioning.scala | 53 +++
.../execution/streaming/statefulOperators.scala | 37 +-
.../commits/0 | 0
.../metadata | 1 +
.../offsets/0 | 3 +
.../state/0/0}/1.delta | Bin
.../state/0/0/_metadata/schema | Bin 0 -> 407 bytes
.../state/0/1/1.delta | Bin 0 -> 96 bytes
.../state/0/2}/1.delta | Bin
.../state/0/3}/1.delta | Bin
.../state/0/4/1.delta | Bin
.../commits/.0.crc | Bin 0 -> 12 bytes
.../commits/.1.crc | Bin 0 -> 12 bytes
.../commits/0 | 0
.../commits/1 | 0
.../metadata | 1 +
.../offsets/.0.crc | Bin 0 -> 16 bytes
.../offsets/.1.crc | Bin 0 -> 16 bytes
.../offsets/0 | 3 +
.../offsets/1 | 3 +
.../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/0}/1.delta | Bin
.../state/0/0}/2.delta | Bin
.../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes
.../state/0/0/_metadata/schema | Bin 0 -> 393 bytes
.../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/1}/1.delta | Bin
.../state/0/1/2.delta | Bin 0 -> 75 bytes
.../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/2}/1.delta | Bin
.../state/0/2/2.delta | Bin 0 -> 75 bytes
.../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/3/1.delta | Bin 0 -> 74 bytes
.../state/0/3/2.delta | Bin
.../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/1.delta | Bin 0 -> 75 bytes
.../state/0/4}/2.delta | Bin
.../commits/.0.crc | Bin 0 -> 12 bytes
.../commits/0 | 0
.../metadata | 1 +
.../offsets/.0.crc | Bin 0 -> 16 bytes
.../offsets/0 | 3 +
.../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/0}/1.delta | Bin
.../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes
.../state/0/0/_metadata/schema | Bin 0 -> 415 bytes
.../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/1.delta | Bin 0 -> 138 bytes
.../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/2}/1.delta | Bin
.../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/3}/1.delta | Bin
.../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/1.delta | Bin
.../commits/.0.crc | Bin 0 -> 12 bytes
.../commits/0 | 0
.../metadata | 1 +
.../offsets/.0.crc | Bin 0 -> 16 bytes
.../offsets/0 | 3 +
.../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/0}/1.delta | Bin
.../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes
.../state/0/0/_metadata/schema | Bin 0 -> 415 bytes
.../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/1.delta | Bin 0 -> 96 bytes
.../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/2}/1.delta | Bin
.../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/3}/1.delta | Bin
.../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/1.delta | Bin
.../commits/.0.crc | Bin 0 -> 12 bytes
.../commits/.1.crc | Bin 0 -> 12 bytes
.../commits/0 | 2 +
.../commits/1 | 2 +
.../metadata | 1 +
.../offsets/.0.crc | Bin 0 -> 16 bytes
.../offsets/.1.crc | Bin 0 -> 16 bytes
.../offsets/0 | 3 +
.../offsets/1 | 3 +
.../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/0}/1.delta | Bin
.../state/0/0}/2.delta | Bin
.../state/0/0/_metadata/.schema.crc | Bin 0 -> 16 bytes
.../state/0/0/_metadata/schema | Bin 0 -> 754 bytes
.../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/1.delta | Bin 0 -> 259 bytes
.../state/0/1}/2.delta | Bin
.../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/2}/1.delta | Bin
.../state/0/2}/2.delta | Bin
.../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/3/1.delta | Bin 0 -> 230 bytes
.../state/0/3/2.delta | Bin
.../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/1.delta | Bin
.../state/0/4}/2.delta | Bin
.../commits/.0.crc | Bin 0 -> 12 bytes
.../commits/.1.crc | Bin 0 -> 12 bytes
.../commits/0 | 0
.../commits/1 | 0
.../metadata | 1 +
.../offsets/.0.crc | Bin 0 -> 16 bytes
.../offsets/.1.crc | Bin 0 -> 16 bytes
.../offsets/0 | 3 +
.../offsets/1 | 3 +
.../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/0}/1.delta | Bin
.../state/0/0}/2.delta | Bin
.../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes
.../state/0/0/_metadata/schema | Bin 0 -> 262 bytes
.../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/1/1.delta | Bin 0 -> 82 bytes
.../state/0/1/2.delta | Bin 0 -> 82 bytes
.../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/2}/1.delta | Bin
.../state/0/2}/2.delta | Bin
.../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/3}/1.delta | Bin
.../state/0/3/2.delta | Bin
.../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes
.../state/0/4/1.delta | Bin
.../state/0/4/2.delta | Bin 0 -> 82 bytes
.../sql/execution/WholeStageCodegenSuite.scala | 2 +-
.../FlatMapGroupsWithStateDistributionSuite.scala | 455 +++++++++++++++++++++
.../StreamingAggregationDistributionSuite.scala | 223 ++++++++++
.../sql/streaming/StreamingAggregationSuite.scala | 37 +-
.../StreamingDeduplicationDistributionSuite.scala | 148 +++++++
.../StreamingSessionWindowDistributionSuite.scala | 225 ++++++++++
...StatefulOpClusteredDistributionTestHelper.scala | 80 ++++
157 files changed, 1393 insertions(+), 86 deletions(-)
create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StatefulOperatorPartitioning.scala
copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition}/commits/0 (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/offsets/0
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/0}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/1/1.delta
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/2}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/3}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition}/state/0/4/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/commits/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/commits/.1.crc
copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-deduplication-with-repartition}/commits/0 (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-test-offsetId-commitId-inconsistent => checkpoint-version-3.2.0-deduplication-with-repartition}/commits/1 (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/.1.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/1
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/_metadata/.schema.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/2.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/2.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/.2.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/1.delta
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-deduplication-with-repartition}/state/0/3/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/.2.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/1.delta
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/commits/.0.crc
copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition}/commits/0 (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/offsets/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/offsets/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/_metadata/.schema.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/1/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/1/1.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/2/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/2}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/3/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/3}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/4/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition}/state/0/4/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/commits/.0.crc
copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition}/commits/0 (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/offsets/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/offsets/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/_metadata/.schema.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/1/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/1/1.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/2/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/2}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/3/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/3}/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/4/.1.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition}/state/0/4/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/.1.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/1
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/.1.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/1
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/0}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/0}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/_metadata/.schema.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/.2.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/1.delta
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/1}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/2/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/2/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/2}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/2}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/.2.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/1.delta
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-session-window-with-repartition}/state/0/3/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/4/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/4/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-session-window-with-repartition}/state/0/4/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/4}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/commits/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/commits/.1.crc
copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/commits/0 (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-test-offsetId-commitId-inconsistent => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/commits/1 (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/metadata
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/.0.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/.1.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/0
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/1
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/_metadata/.schema.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/_metadata/schema
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/.2.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/1.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/2.delta
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2}/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3}/1.delta (100%)
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/state/0/3/2.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/.1.delta.crc
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/.2.delta.crc
copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/state/0/4/1.delta (100%)
create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/2.delta
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationDistributionSuite.scala
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowDistributionSuite.scala
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/util/StatefulOpClusteredDistributionTestHelper.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org