You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/04/21 19:13:15 UTC

[incubator-iceberg] branch v2-spec updated (7fcc6ee -> 5983a49)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a change to branch v2-spec
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git.


    from 7fcc6ee  Current work.
     add 45a44f4  ORC: Refactor readers to remove duplicate null handling code (#899)
     add d8dfb27  Use only guava Preconditions (#911)
     add bedc9c7  Spark: Implement an action to remove orphan files (#894)
     add 720b240  Update Snapshot and TableMetadata with sequence numbers (#910)
     add 3334afc  Remove unused fields from DataFile (#914)
     add bde47f3  Add missing @Override annotations (#916)
     add 3a35c07  Parquet: Support constant map for partition values (#909)
     add a4f1619  HOTFIX: Fix partition constants (#918)
     add 487fc1c  Add v2 manifest lists (#907)
     add 7b99b17  Fix metadata tables with staged initial snapshots (#908)
     add b6cdc69  Add persistent IDs to partition fields (#845)
     add 409efd0  Fix the size in GenericDataFile (#929)
     add 8b27fae  Spark: move actions to a separate package (#927)
     add a5e7736  Move manifest factory methods (#925)
     add e0a3304  Add void transform that always produces null (#924)
     add 98fef8b  Remove unused ManifestLists.write method (#921)
     add e8e83ca  Bump Apache Spark to 2.4.5 (#917)
     add e30e304  Improve failure handling after Snapshot expiration (#928)
     add d1f991f  Spark: limit the listing depth in RemoveOrphanFilesAction (#930)
     add 34126d2  Add SupportsNamespaces extension to Catalog (#919)
     add af25cbe  Add residual evaluation for MR reader (#931)
     add fdca3f8  Add v2 manifests (#913)
     add 3160518  Spark: Implement an adapter to wrap Row into DataFile (#937)
     add bd30a35  Spark: Use SparkValueConverter in TestSparkReadProjection (#942)
     add 7319581  Update dev/source-release.sh to add arrow, mr, and spark-runtime (#944)
     add 6840013  Log test failure exceptions in Travis CI. (#945)
     add 83230c2  Fix thread safety in listeners and simplify implementation (#940)
     add ffa1882  Document Java version should be 1.8 (#938)
     add 5983a49  Merge branch 'master' into v2-spec

No new revisions were added by this update.

Summary of changes:
 README.md                                          |   2 +-
 api/src/main/java/org/apache/iceberg/DataFile.java |  12 -
 .../main/java/org/apache/iceberg/ManifestFile.java |  59 ++-
 .../java/org/apache/iceberg/PartitionField.java    |  16 +-
 .../java/org/apache/iceberg/PartitionSpec.java     |  69 ++-
 api/src/main/java/org/apache/iceberg/Snapshot.java |   9 +
 .../apache/iceberg/catalog/SupportsNamespaces.java | 141 ++++++
 .../java/org/apache/iceberg/events/Listeners.java  |  27 +-
 ...eption.java => NamespaceNotEmptyException.java} |   8 +-
 ...xception.java => NoSuchNamespaceException.java} |   8 +-
 .../org/apache/iceberg/transforms/Identity.java    |   5 +
 .../SerializationProxies.java}                     |  35 +-
 .../org/apache/iceberg/transforms/Transform.java   |   9 +
 .../org/apache/iceberg/transforms/Transforms.java  |  14 +
 .../apache/iceberg/transforms/VoidTransform.java   |  76 ++++
 .../org/apache/iceberg/PartitionSpecTestBase.java  |  76 ++++
 .../test/java/org/apache/iceberg/TestHelpers.java  |  20 +-
 .../iceberg/TestPartitionSpecValidation.java       |  45 +-
 .../apache/iceberg/TestTransformSerialization.java |  54 +--
 .../parquet/BaseVectorizedParquetValuesReader.java |   2 +-
 .../parquet/VectorizedColumnIterator.java          |   2 +-
 build.gradle                                       |   9 +-
 .../java/org/apache/iceberg/AllDataFilesTable.java |   2 +-
 .../java/org/apache/iceberg/AllEntriesTable.java   |   2 +-
 .../java/org/apache/iceberg/AllManifestsTable.java |   2 +-
 ...ableScan.java => BaseAllMetadataTableScan.java} |  40 +-
 .../org/apache/iceberg/BaseMetastoreCatalog.java   |   2 +-
 .../org/apache/iceberg/BaseRewriteManifests.java   |   7 +-
 .../main/java/org/apache/iceberg/BaseSnapshot.java |  28 +-
 .../java/org/apache/iceberg/DataFilesTable.java    |   2 +-
 .../main/java/org/apache/iceberg/FastAppend.java   |   5 +-
 .../java/org/apache/iceberg/GenericDataFile.java   |  50 +--
 .../org/apache/iceberg/GenericManifestEntry.java   | 180 ++++++++
 .../org/apache/iceberg/GenericManifestFile.java    |  89 ++--
 ...StreamingDelete.java => IndexedStructLike.java} |  49 +-
 .../apache/iceberg/InheritableMetadataFactory.java |  27 +-
 .../java/org/apache/iceberg/ManifestEntry.java     | 361 ++-------------
 .../java/org/apache/iceberg/ManifestFiles.java     | 145 ++++++
 .../java/org/apache/iceberg/ManifestGroup.java     |   2 +-
 .../org/apache/iceberg/ManifestListWriter.java     |  95 ++--
 .../java/org/apache/iceberg/ManifestLists.java     |  65 +++
 .../java/org/apache/iceberg/ManifestReader.java    |  69 +--
 .../java/org/apache/iceberg/ManifestWriter.java    | 165 ++++---
 .../apache/iceberg/MergingSnapshotProducer.java    |   9 +-
 .../org/apache/iceberg/PartitionSpecParser.java    |  15 +-
 .../java/org/apache/iceberg/RemoveSnapshots.java   | 214 ++++-----
 .../java/org/apache/iceberg/SnapshotParser.java    |  14 +-
 .../java/org/apache/iceberg/SnapshotProducer.java  |  19 +-
 .../java/org/apache/iceberg/TableMetadata.java     |  88 +++-
 .../org/apache/iceberg/TableMetadataParser.java    |  12 +-
 .../main/java/org/apache/iceberg/V1Metadata.java   | 410 +++++++++++++++++
 .../main/java/org/apache/iceberg/V2Metadata.java   | 336 ++++++++++++++
 .../java/org/apache/iceberg/avro/ValueReaders.java |  10 +-
 .../org/apache/iceberg/hadoop/HadoopCatalog.java   | 126 +++++-
 .../apache/iceberg/hadoop/HiddenPathFilter.java    |  32 +-
 .../java/org/apache/iceberg/util/JsonUtil.java     |   2 +-
 .../org/apache/iceberg/util/PartitionUtil.java     |  18 +-
 .../java/org/apache/iceberg/TableTestBase.java     |  18 +-
 .../apache/iceberg/TestGenericManifestFile.java    |  91 ----
 .../apache/iceberg/TestManifestListVersions.java   | 215 +++++++++
 .../org/apache/iceberg/TestManifestReader.java     |  47 ++
 .../apache/iceberg/TestManifestWriterVersions.java | 236 ++++++++++
 .../java/org/apache/iceberg/TestMergeAppend.java   |  95 ++++
 .../org/apache/iceberg/TestPartitionSpecInfo.java  |   4 +-
 .../apache/iceberg/TestPartitionSpecParser.java    | 122 +++++
 .../org/apache/iceberg/TestRewriteManifests.java   |  14 +-
 .../java/org/apache/iceberg/TestSnapshotJson.java  |   8 +-
 .../java/org/apache/iceberg/TestTableMetadata.java |  78 +++-
 .../java/org/apache/iceberg/TestTransaction.java   |   2 +-
 .../apache/iceberg/hadoop/TestHadoopCatalog.java   | 168 ++++++-
 .../java/org/apache/iceberg/data/DateTimeUtil.java |  52 +++
 .../org/apache/iceberg/data/TableScanIterable.java |  38 +-
 .../apache/iceberg/data/avro/GenericReaders.java   |  15 +-
 .../apache/iceberg/data/orc/GenericOrcReader.java  | 247 +++--------
 .../apache/iceberg/data/orc/GenericOrcWriter.java  |   3 +
 .../data/parquet/GenericParquetReaders.java        |  38 +-
 dev/source-release.sh                              |   2 +-
 .../java/org/apache/iceberg/hive/HiveCatalog.java  | 221 ++++++++-
 .../org/apache/iceberg/hive/TestHiveCatalog.java   | 192 ++++++++
 .../iceberg/mr/mapreduce/IcebergInputFormat.java   |  28 +-
 .../mr/mapreduce/TestIcebergInputFormat.java       |  73 ++-
 .../apache/iceberg/parquet/MessageTypeToType.java  |   8 +-
 .../org/apache/iceberg/parquet/PageIterator.java   |   1 +
 site/docs/spec.md                                  |  30 +-
 .../java/org/apache/iceberg/actions/Action.java    |  18 +-
 .../java/org/apache/iceberg/actions/Actions.java}  |  30 +-
 .../iceberg/actions/RemoveOrphanFilesAction.java   | 289 ++++++++++++
 .../org/apache/iceberg/spark/SparkDataFile.java    | 156 +++++++
 .../org/apache/iceberg/spark/SparkStructLike.java  |  39 +-
 .../apache/iceberg/spark/SparkValueConverter.java  | 119 +++++
 .../apache/iceberg/spark/data/SparkOrcReader.java  | 376 +++-------------
 .../iceberg/spark/data/SparkParquetReaders.java    |  39 +-
 .../iceberg/spark/data/SparkValueReaders.java      |  28 --
 .../iceberg/spark/source/BaseDataReader.java       |   2 +-
 .../apache/iceberg/spark/source/RowDataReader.java |  47 +-
 .../org/apache/iceberg/spark/SparkTableUtil.scala  |   4 +-
 .../iceberg/TestManifestFileSerialization.java     |   2 +-
 .../actions/TestRemoveOrphanFilesAction.java       | 494 +++++++++++++++++++++
 .../apache/iceberg/spark/TestSparkDataFile.java    | 208 +++++++++
 .../spark/source/TestForwardCompatibility.java     |   8 +-
 .../spark/source/TestIcebergSourceHiveTables.java  |  15 +-
 .../spark/source/TestIcebergSourceTablesBase.java  |  87 ++++
 .../iceberg/spark/source/TestPartitionValues.java  |  69 +++
 .../iceberg/spark/source/TestSparkDataWrite.java   |   8 +-
 .../spark/source/TestSparkReadProjection.java      |  89 +---
 versions.lock                                      |  28 +-
 versions.props                                     |   4 +-
 107 files changed, 5728 insertions(+), 1867 deletions(-)
 create mode 100644 api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java
 copy api/src/main/java/org/apache/iceberg/exceptions/{NoSuchTableException.java => NamespaceNotEmptyException.java} (76%)
 copy api/src/main/java/org/apache/iceberg/exceptions/{CommitFailedException.java => NoSuchNamespaceException.java} (76%)
 copy api/src/main/java/org/apache/iceberg/{types/PrimitiveHolder.java => transforms/SerializationProxies.java} (57%)
 create mode 100644 api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
 create mode 100644 api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java
 copy core/src/main/java/org/apache/iceberg/{StaticTableScan.java => BaseAllMetadataTableScan.java} (51%)
 create mode 100644 core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
 copy core/src/main/java/org/apache/iceberg/{StreamingDelete.java => IndexedStructLike.java} (52%)
 create mode 100644 core/src/main/java/org/apache/iceberg/ManifestFiles.java
 create mode 100644 core/src/main/java/org/apache/iceberg/ManifestLists.java
 create mode 100644 core/src/main/java/org/apache/iceberg/V1Metadata.java
 create mode 100644 core/src/main/java/org/apache/iceberg/V2Metadata.java
 copy api/src/main/java/org/apache/iceberg/expressions/True.java => core/src/main/java/org/apache/iceberg/hadoop/HiddenPathFilter.java (58%)
 delete mode 100644 core/src/test/java/org/apache/iceberg/TestGenericManifestFile.java
 create mode 100644 core/src/test/java/org/apache/iceberg/TestManifestListVersions.java
 create mode 100644 core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
 create mode 100644 core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java
 create mode 100644 data/src/main/java/org/apache/iceberg/data/DateTimeUtil.java
 create mode 100644 hive/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
 copy orc/src/main/java/org/apache/iceberg/orc/OrcValueReader.java => spark/src/main/java/org/apache/iceberg/actions/Action.java (76%)
 copy spark/src/{test/java/org/apache/iceberg/spark/source/TestIcebergSource.java => main/java/org/apache/iceberg/actions/Actions.java} (59%)
 create mode 100644 spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
 create mode 100644 spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java
 copy api/src/main/java/org/apache/iceberg/expressions/And.java => spark/src/main/java/org/apache/iceberg/spark/SparkStructLike.java (53%)
 create mode 100644 spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java
 create mode 100644 spark/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java
 create mode 100644 spark/src/test/java/org/apache/iceberg/spark/TestSparkDataFile.java