You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/06/30 00:56:15 UTC
[iceberg] branch master updated (e168f59 -> 51c930e)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git.
from e168f59 Parquet: Fix executor memory leak in row group filter (#1139)
add 51c930e Spark: Add Spark 3 data source classes (#1124)
No new revisions were added by this update.
Summary of changes:
build.gradle | 1 +
.../org/apache/iceberg/hadoop/HadoopInputFile.java | 23 +
.../main/java/org/apache/iceberg/hadoop/Util.java | 15 +
.../java/org/apache/iceberg/hive/HiveCatalog.java | 12 +
.../org/apache/iceberg/actions/BaseAction.java | 13 -
.../iceberg/actions/RewriteDataFilesAction.java | 3 +-
.../iceberg/actions/RewriteManifestsAction.java | 3 +-
.../BaseAction.java => spark/SparkUtil.java} | 45 +-
.../apache/iceberg/spark/source/LogMessage.java | 0
.../apache/iceberg/spark/source/SimpleRecord.java | 0
.../apache/iceberg/spark/source/TestAvroScan.java | 2 +-
.../iceberg/spark/source/TestDataFrameWrites.java | 18 +-
.../spark/source/TestDataSourceOptions.java | 8 +-
.../spark/source/TestForwardCompatibility.java | 13 +-
.../spark/source/TestIdentityPartitionData.java | 2 +-
.../iceberg/spark/source/TestParquetScan.java | 2 +-
.../iceberg/spark/source/TestPartitionValues.java | 193 ++++----
.../iceberg/spark/source/TestReadProjection.java | 0
.../spark/source/TestSnapshotSelection.java | 2 +-
.../iceberg/spark/source}/TestSparkDataFile.java | 6 +-
.../iceberg/spark/source/TestSparkDataWrite.java | 12 +-
.../spark/source/TestSparkReadProjection.java | 2 +-
.../iceberg/spark/source/TestSparkSchema.java | 2 +-
.../spark/source/TestStructuredStreaming.java | 41 +-
.../apache/iceberg/spark/source/TestTables.java | 0
.../spark/source/TestWriteMetricsConfig.java | 2 +-
.../apache/iceberg/spark/source/IcebergSource.java | 41 +-
.../iceberg/spark/source/TestAvroScan24.java} | 4 +-
.../spark/source/TestDataFrameWrites24.java} | 7 +-
.../spark/source/TestDataSourceOptions24.java} | 4 +-
.../spark/source/TestForwardCompatibility24.java} | 25 +-
.../spark/source/TestIdentityPartitionData24.java} | 7 +-
.../iceberg/spark/source/TestParquetScan24.java} | 7 +-
.../spark/source/TestPartitionValues24.java} | 7 +-
.../spark/source/TestSnapshotSelection24.java} | 4 +-
.../iceberg/spark/source/TestSparkDataFile24.java} | 4 +-
.../spark/source/TestSparkDataWrite24.java} | 7 +-
.../spark/source/TestSparkReadProjection24.java} | 7 +-
.../iceberg/spark/source/TestSparkSchema24.java} | 4 +-
.../spark/source/TestStructuredStreaming24.java} | 25 +-
.../spark/source/TestWriteMetricsConfig24.java} | 4 +-
spark3/dependencies.lock | 110 ++---
.../java/org/apache/iceberg/spark/Spark3Util.java | 510 +++++++++++++++++++++
.../org/apache/iceberg/spark/SparkCatalog.java | 282 ++++++++++++
.../org/apache/iceberg/spark/SparkFilters.java | 24 +-
.../apache/iceberg/spark/SparkSessionCatalog.java | 188 ++++++++
.../apache/iceberg/spark/source/IcebergSource.java | 103 +++++
.../iceberg/spark/source/SparkBatchScan.java | 370 +++++++++++++++
.../iceberg/spark/source/SparkBatchWrite.java | 115 +++--
.../iceberg/spark/source/SparkScanBuilder.java | 144 ++++++
.../iceberg/spark/source/SparkStreamingWrite.java | 37 +-
.../apache/iceberg/spark/source/SparkTable.java | 176 +++++++
.../iceberg/spark/source/SparkWriteBuilder.java | 161 +++++++
.../iceberg/spark/source/StagedSparkTable.java | 23 +-
.../org/apache/iceberg/spark/source/Stats.java | 2 +-
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/iceberg/spark/source/TestAvroScan3.java | 4 +-
.../iceberg/spark/source/TestDataFrameWrites3.java | 7 +-
.../spark/source/TestDataSourceOptions3.java | 4 +-
.../iceberg/spark/source/TestFilteredScan.java | 295 +++++-------
.../spark/source/TestForwardCompatibility3.java | 26 +-
.../iceberg/spark/source/TestIcebergSource.java | 6 +-
.../spark/source/TestIdentityPartitionData3.java | 7 +-
.../iceberg/spark/source/TestParquetScan3.java | 7 +-
.../iceberg/spark/source/TestPartitionValues3.java | 7 +-
.../spark/source/TestSnapshotSelection3.java | 4 +-
.../iceberg/spark/source/TestSparkDataFile3.java | 4 +-
.../iceberg/spark/source/TestSparkDataWrite3.java | 7 +-
.../spark/source/TestSparkReadProjection3.java | 7 +-
.../iceberg/spark/source/TestSparkSchema3.java | 4 +-
.../spark/source/TestStructuredStreaming3.java | 26 +-
.../spark/source/TestWriteMetricsConfig3.java | 4 +-
72 files changed, 2570 insertions(+), 671 deletions(-)
copy spark/src/main/java/org/apache/iceberg/{actions/BaseAction.java => spark/SparkUtil.java} (54%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/LogMessage.java (100%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java (100%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java (98%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java (97%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java (98%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java (94%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java (99%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java (98%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java (75%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java (100%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java (99%)
rename {spark2/src/test/java/org/apache/iceberg/spark => spark/src/test/java/org/apache/iceberg/spark/source}/TestSparkDataFile.java (97%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java (97%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java (99%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java (99%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java (89%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestTables.java (100%)
rename {spark2 => spark}/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java (99%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestAvroScan24.java} (93%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java} (86%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java} (92%)
copy spark2/src/{main/java/org/apache/iceberg/spark/source/Stats.java => test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java} (61%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java} (84%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestParquetScan24.java} (86%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java} (86%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java} (92%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java} (93%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java} (86%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java} (82%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java} (93%)
copy spark2/src/{main/java/org/apache/iceberg/spark/source/Stats.java => test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java} (61%)
copy spark2/src/{jmh/java/org/apache/iceberg/spark/source/Action.java => test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java} (92%)
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
copy {spark2 => spark3}/src/main/java/org/apache/iceberg/spark/SparkFilters.java (88%)
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java
copy spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java => spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchWrite.java (68%)
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
copy spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java => spark3/src/main/java/org/apache/iceberg/spark/source/SparkStreamingWrite.java (73%)
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
create mode 100644 spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
copy spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java => spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java (65%)
copy {spark2 => spark3}/src/main/java/org/apache/iceberg/spark/source/Stats.java (95%)
copy {spark2 => spark3}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java (93%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java (86%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java (92%)
copy {spark2 => spark3}/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java (61%)
copy spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java (59%)
copy {spark2 => spark3}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java (84%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java (84%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java (86%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java (86%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java (92%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java (93%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java (86%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java (82%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema3.java (93%)
copy spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java (59%)
copy spark2/src/jmh/java/org/apache/iceberg/spark/source/Action.java => spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java (92%)