You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/04/21 19:13:15 UTC
[incubator-iceberg] branch v2-spec updated (7fcc6ee -> 5983a49)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a change to branch v2-spec
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git.
from 7fcc6ee Current work.
add 45a44f4 ORC: Refactor readers to remove duplicate null handling code (#899)
add d8dfb27 Use only guava Preconditions (#911)
add bedc9c7 Spark: Implement an action to remove orphan files (#894)
add 720b240 Update Snapshot and TableMetadata with sequence numbers (#910)
add 3334afc Remove unused fields from DataFile (#914)
add bde47f3 Add missing @Override annotations (#916)
add 3a35c07 Parquet: Support constant map for partition values (#909)
add a4f1619 HOTFIX: Fix partition constants (#918)
add 487fc1c Add v2 manifest lists (#907)
add 7b99b17 Fix metadata tables with staged initial snapshots (#908)
add b6cdc69 Add persistent IDs to partition fields (#845)
add 409efd0 Fix the size in GenericDataFile (#929)
add 8b27fae Spark: move actions to a separate package (#927)
add a5e7736 Move manifest factory methods (#925)
add e0a3304 Add void transform that always produces null (#924)
add 98fef8b Remove unused ManifestLists.write method (#921)
add e8e83ca Bump Apache Spark to 2.4.5 (#917)
add e30e304 Improve failure handling after Snapshot expiration (#928)
add d1f991f Spark: limit the listing depth in RemoveOrphanFilesAction (#930)
add 34126d2 Add SupportsNamespaces extension to Catalog (#919)
add af25cbe Add residual evaluation for MR reader (#931)
add fdca3f8 Add v2 manifests (#913)
add 3160518 Spark: Implement an adapter to wrap Row into DataFile (#937)
add bd30a35 Spark: Use SparkValueConverter in TestSparkReadProjection (#942)
add 7319581 Update dev/source-release.sh to add arrow, mr, and spark-runtime (#944)
add 6840013 Log test failure exceptions in Travis CI. (#945)
add 83230c2 Fix thread safety in listeners and simplify implementation (#940)
add ffa1882 Document Java version should be 1.8 (#938)
add 5983a49 Merge branch 'master' into v2-spec
No new revisions were added by this update.
Summary of changes:
README.md | 2 +-
api/src/main/java/org/apache/iceberg/DataFile.java | 12 -
.../main/java/org/apache/iceberg/ManifestFile.java | 59 ++-
.../java/org/apache/iceberg/PartitionField.java | 16 +-
.../java/org/apache/iceberg/PartitionSpec.java | 69 ++-
api/src/main/java/org/apache/iceberg/Snapshot.java | 9 +
.../apache/iceberg/catalog/SupportsNamespaces.java | 141 ++++++
.../java/org/apache/iceberg/events/Listeners.java | 27 +-
...eption.java => NamespaceNotEmptyException.java} | 8 +-
...xception.java => NoSuchNamespaceException.java} | 8 +-
.../org/apache/iceberg/transforms/Identity.java | 5 +
.../SerializationProxies.java} | 35 +-
.../org/apache/iceberg/transforms/Transform.java | 9 +
.../org/apache/iceberg/transforms/Transforms.java | 14 +
.../apache/iceberg/transforms/VoidTransform.java | 76 ++++
.../org/apache/iceberg/PartitionSpecTestBase.java | 76 ++++
.../test/java/org/apache/iceberg/TestHelpers.java | 20 +-
.../iceberg/TestPartitionSpecValidation.java | 45 +-
.../apache/iceberg/TestTransformSerialization.java | 54 +--
.../parquet/BaseVectorizedParquetValuesReader.java | 2 +-
.../parquet/VectorizedColumnIterator.java | 2 +-
build.gradle | 9 +-
.../java/org/apache/iceberg/AllDataFilesTable.java | 2 +-
.../java/org/apache/iceberg/AllEntriesTable.java | 2 +-
.../java/org/apache/iceberg/AllManifestsTable.java | 2 +-
...ableScan.java => BaseAllMetadataTableScan.java} | 40 +-
.../org/apache/iceberg/BaseMetastoreCatalog.java | 2 +-
.../org/apache/iceberg/BaseRewriteManifests.java | 7 +-
.../main/java/org/apache/iceberg/BaseSnapshot.java | 28 +-
.../java/org/apache/iceberg/DataFilesTable.java | 2 +-
.../main/java/org/apache/iceberg/FastAppend.java | 5 +-
.../java/org/apache/iceberg/GenericDataFile.java | 50 +--
.../org/apache/iceberg/GenericManifestEntry.java | 180 ++++++++
.../org/apache/iceberg/GenericManifestFile.java | 89 ++--
...StreamingDelete.java => IndexedStructLike.java} | 49 +-
.../apache/iceberg/InheritableMetadataFactory.java | 27 +-
.../java/org/apache/iceberg/ManifestEntry.java | 361 ++-------------
.../java/org/apache/iceberg/ManifestFiles.java | 145 ++++++
.../java/org/apache/iceberg/ManifestGroup.java | 2 +-
.../org/apache/iceberg/ManifestListWriter.java | 95 ++--
.../java/org/apache/iceberg/ManifestLists.java | 65 +++
.../java/org/apache/iceberg/ManifestReader.java | 69 +--
.../java/org/apache/iceberg/ManifestWriter.java | 165 ++++---
.../apache/iceberg/MergingSnapshotProducer.java | 9 +-
.../org/apache/iceberg/PartitionSpecParser.java | 15 +-
.../java/org/apache/iceberg/RemoveSnapshots.java | 214 ++++-----
.../java/org/apache/iceberg/SnapshotParser.java | 14 +-
.../java/org/apache/iceberg/SnapshotProducer.java | 19 +-
.../java/org/apache/iceberg/TableMetadata.java | 88 +++-
.../org/apache/iceberg/TableMetadataParser.java | 12 +-
.../main/java/org/apache/iceberg/V1Metadata.java | 410 +++++++++++++++++
.../main/java/org/apache/iceberg/V2Metadata.java | 336 ++++++++++++++
.../java/org/apache/iceberg/avro/ValueReaders.java | 10 +-
.../org/apache/iceberg/hadoop/HadoopCatalog.java | 126 +++++-
.../apache/iceberg/hadoop/HiddenPathFilter.java | 32 +-
.../java/org/apache/iceberg/util/JsonUtil.java | 2 +-
.../org/apache/iceberg/util/PartitionUtil.java | 18 +-
.../java/org/apache/iceberg/TableTestBase.java | 18 +-
.../apache/iceberg/TestGenericManifestFile.java | 91 ----
.../apache/iceberg/TestManifestListVersions.java | 215 +++++++++
.../org/apache/iceberg/TestManifestReader.java | 47 ++
.../apache/iceberg/TestManifestWriterVersions.java | 236 ++++++++++
.../java/org/apache/iceberg/TestMergeAppend.java | 95 ++++
.../org/apache/iceberg/TestPartitionSpecInfo.java | 4 +-
.../apache/iceberg/TestPartitionSpecParser.java | 122 +++++
.../org/apache/iceberg/TestRewriteManifests.java | 14 +-
.../java/org/apache/iceberg/TestSnapshotJson.java | 8 +-
.../java/org/apache/iceberg/TestTableMetadata.java | 78 +++-
.../java/org/apache/iceberg/TestTransaction.java | 2 +-
.../apache/iceberg/hadoop/TestHadoopCatalog.java | 168 ++++++-
.../java/org/apache/iceberg/data/DateTimeUtil.java | 52 +++
.../org/apache/iceberg/data/TableScanIterable.java | 38 +-
.../apache/iceberg/data/avro/GenericReaders.java | 15 +-
.../apache/iceberg/data/orc/GenericOrcReader.java | 247 +++--------
.../apache/iceberg/data/orc/GenericOrcWriter.java | 3 +
.../data/parquet/GenericParquetReaders.java | 38 +-
dev/source-release.sh | 2 +-
.../java/org/apache/iceberg/hive/HiveCatalog.java | 221 ++++++++-
.../org/apache/iceberg/hive/TestHiveCatalog.java | 192 ++++++++
.../iceberg/mr/mapreduce/IcebergInputFormat.java | 28 +-
.../mr/mapreduce/TestIcebergInputFormat.java | 73 ++-
.../apache/iceberg/parquet/MessageTypeToType.java | 8 +-
.../org/apache/iceberg/parquet/PageIterator.java | 1 +
site/docs/spec.md | 30 +-
.../java/org/apache/iceberg/actions/Action.java | 18 +-
.../java/org/apache/iceberg/actions/Actions.java} | 30 +-
.../iceberg/actions/RemoveOrphanFilesAction.java | 289 ++++++++++++
.../org/apache/iceberg/spark/SparkDataFile.java | 156 +++++++
.../org/apache/iceberg/spark/SparkStructLike.java | 39 +-
.../apache/iceberg/spark/SparkValueConverter.java | 119 +++++
.../apache/iceberg/spark/data/SparkOrcReader.java | 376 +++-------------
.../iceberg/spark/data/SparkParquetReaders.java | 39 +-
.../iceberg/spark/data/SparkValueReaders.java | 28 --
.../iceberg/spark/source/BaseDataReader.java | 2 +-
.../apache/iceberg/spark/source/RowDataReader.java | 47 +-
.../org/apache/iceberg/spark/SparkTableUtil.scala | 4 +-
.../iceberg/TestManifestFileSerialization.java | 2 +-
.../actions/TestRemoveOrphanFilesAction.java | 494 +++++++++++++++++++++
.../apache/iceberg/spark/TestSparkDataFile.java | 208 +++++++++
.../spark/source/TestForwardCompatibility.java | 8 +-
.../spark/source/TestIcebergSourceHiveTables.java | 15 +-
.../spark/source/TestIcebergSourceTablesBase.java | 87 ++++
.../iceberg/spark/source/TestPartitionValues.java | 69 +++
.../iceberg/spark/source/TestSparkDataWrite.java | 8 +-
.../spark/source/TestSparkReadProjection.java | 89 +---
versions.lock | 28 +-
versions.props | 4 +-
107 files changed, 5728 insertions(+), 1867 deletions(-)
create mode 100644 api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java
copy api/src/main/java/org/apache/iceberg/exceptions/{NoSuchTableException.java => NamespaceNotEmptyException.java} (76%)
copy api/src/main/java/org/apache/iceberg/exceptions/{CommitFailedException.java => NoSuchNamespaceException.java} (76%)
copy api/src/main/java/org/apache/iceberg/{types/PrimitiveHolder.java => transforms/SerializationProxies.java} (57%)
create mode 100644 api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
create mode 100644 api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java
copy core/src/main/java/org/apache/iceberg/{StaticTableScan.java => BaseAllMetadataTableScan.java} (51%)
create mode 100644 core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
copy core/src/main/java/org/apache/iceberg/{StreamingDelete.java => IndexedStructLike.java} (52%)
create mode 100644 core/src/main/java/org/apache/iceberg/ManifestFiles.java
create mode 100644 core/src/main/java/org/apache/iceberg/ManifestLists.java
create mode 100644 core/src/main/java/org/apache/iceberg/V1Metadata.java
create mode 100644 core/src/main/java/org/apache/iceberg/V2Metadata.java
copy api/src/main/java/org/apache/iceberg/expressions/True.java => core/src/main/java/org/apache/iceberg/hadoop/HiddenPathFilter.java (58%)
delete mode 100644 core/src/test/java/org/apache/iceberg/TestGenericManifestFile.java
create mode 100644 core/src/test/java/org/apache/iceberg/TestManifestListVersions.java
create mode 100644 core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
create mode 100644 core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java
create mode 100644 data/src/main/java/org/apache/iceberg/data/DateTimeUtil.java
create mode 100644 hive/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
copy orc/src/main/java/org/apache/iceberg/orc/OrcValueReader.java => spark/src/main/java/org/apache/iceberg/actions/Action.java (76%)
copy spark/src/{test/java/org/apache/iceberg/spark/source/TestIcebergSource.java => main/java/org/apache/iceberg/actions/Actions.java} (59%)
create mode 100644 spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
create mode 100644 spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java
copy api/src/main/java/org/apache/iceberg/expressions/And.java => spark/src/main/java/org/apache/iceberg/spark/SparkStructLike.java (53%)
create mode 100644 spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java
create mode 100644 spark/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java
create mode 100644 spark/src/test/java/org/apache/iceberg/spark/TestSparkDataFile.java