You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/08/11 16:58:34 UTC

[iceberg] branch 0.9.x created (now 768dc08)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a change to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git.


      at 768dc08  ORC: Fix decimal and timestamp bugs (#1271)

This branch includes the following new commits:

     new 194407a  Spark: Fix USING clause in SparkCatalog (#1194)
     new ecebbcd  Remove table cache expiration (#1203)
     new 505ee61  Spark: Fix import for paths that include spaces (#1228)
     new b012d67  Fix Guava imports (#1251)
     new 3f68359  Ban imports from org.apache.commons.compress.utils (#1278)
     new 37a5709  Implement negate in bound predicates (#1282)
     new f3242b4  Spark: Fix Spark 3 describe with IN predicates (#1283)
     new e116345  Core: Use array instead of list in BaseCombinedScanTask to fix serialization (#1285)
     new 768dc08  ORC: Fix decimal and timestamp bugs (#1271)

The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[iceberg] 02/09: Remove table cache expiration (#1203)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit ecebbcd796046bdd5bf13114bd96bf7db398bbcd
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Thu Jul 16 16:43:29 2020 -0700

    Remove table cache expiration (#1203)
---
 core/src/main/java/org/apache/iceberg/CachingCatalog.java | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/CachingCatalog.java b/core/src/main/java/org/apache/iceberg/CachingCatalog.java
index ab9f90f..b3ab1fa 100644
--- a/core/src/main/java/org/apache/iceberg/CachingCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/CachingCatalog.java
@@ -23,7 +23,6 @@ import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.catalog.Namespace;
@@ -40,10 +39,7 @@ public class CachingCatalog implements Catalog {
     return new CachingCatalog(catalog, caseSensitive);
   }
 
-  private final Cache<TableIdentifier, Table> tableCache = Caffeine.newBuilder()
-      .softValues()
-      .expireAfterAccess(1, TimeUnit.MINUTES)
-      .build();
+  private final Cache<TableIdentifier, Table> tableCache = Caffeine.newBuilder().softValues().build();
   private final Catalog catalog;
   private final boolean caseSensitive;
 


[iceberg] 07/09: Spark: Fix Spark 3 describe with IN predicates (#1283)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit f3242b4394063a1899be096d4cb027edc2d47d11
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Jul 31 15:57:05 2020 -0700

    Spark: Fix Spark 3 describe with IN predicates (#1283)
---
 spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 03d39be..03a4ea5 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.TableProperties;
@@ -517,11 +518,19 @@ public class Spark3Util {
           return pred.ref().name() + " != " + sqlString(pred.literal());
         case STARTS_WITH:
           return pred.ref().name() + " LIKE '" + pred.literal() + "%'";
+        case IN:
+          return pred.ref().name() + " IN (" + sqlString(pred.literals()) + ")";
+        case NOT_IN:
+          return pred.ref().name() + " NOT IN (" + sqlString(pred.literals()) + ")";
         default:
           throw new UnsupportedOperationException("Cannot convert predicate to SQL: " + pred);
       }
     }
 
+    private static <T> String sqlString(List<org.apache.iceberg.expressions.Literal<T>> literals) {
+      return literals.stream().map(DescribeExpressionVisitor::sqlString).collect(Collectors.joining(", "));
+    }
+
     private static String sqlString(org.apache.iceberg.expressions.Literal<?> lit) {
       if (lit.value() instanceof String) {
         return "'" + lit.value() + "'";


[iceberg] 01/09: Spark: Fix USING clause in SparkCatalog (#1194)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit 194407ae1ed2053a4187f40e6bce9d83f1b885e0
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Tue Jul 14 09:37:54 2020 -0700

    Spark: Fix USING clause in SparkCatalog (#1194)
---
 .../org/apache/iceberg/spark/SparkCatalog.java     | 24 ++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
index 86b8bdb..c115b84 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
@@ -149,8 +149,12 @@ public class SparkCatalog implements StagingTableCatalog, org.apache.spark.sql.c
                                  Map<String, String> properties) throws TableAlreadyExistsException {
     Schema icebergSchema = SparkSchemaUtil.convert(schema);
     try {
-      return new StagedSparkTable(icebergCatalog.newCreateTableTransaction(buildIdentifier(ident), icebergSchema,
-          Spark3Util.toPartitionSpec(icebergSchema, transforms), properties.get("location"), properties));
+      return new StagedSparkTable(icebergCatalog.newCreateTableTransaction(
+          buildIdentifier(ident),
+          icebergSchema,
+          Spark3Util.toPartitionSpec(icebergSchema, transforms),
+          properties.get("location"),
+          Spark3Util.rebuildCreateProperties(properties)));
     } catch (AlreadyExistsException e) {
       throw new TableAlreadyExistsException(ident);
     }
@@ -161,8 +165,12 @@ public class SparkCatalog implements StagingTableCatalog, org.apache.spark.sql.c
                                   Map<String, String> properties) throws NoSuchTableException {
     Schema icebergSchema = SparkSchemaUtil.convert(schema);
     try {
-      return new StagedSparkTable(icebergCatalog.newReplaceTableTransaction(buildIdentifier(ident), icebergSchema,
-          Spark3Util.toPartitionSpec(icebergSchema, transforms), properties.get("location"), properties,
+      return new StagedSparkTable(icebergCatalog.newReplaceTableTransaction(
+          buildIdentifier(ident),
+          icebergSchema,
+          Spark3Util.toPartitionSpec(icebergSchema, transforms),
+          properties.get("location"),
+          Spark3Util.rebuildCreateProperties(properties),
           false /* do not create */));
     } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
       throw new NoSuchTableException(ident);
@@ -173,8 +181,12 @@ public class SparkCatalog implements StagingTableCatalog, org.apache.spark.sql.c
   public StagedTable stageCreateOrReplace(Identifier ident, StructType schema, Transform[] transforms,
                                           Map<String, String> properties) {
     Schema icebergSchema = SparkSchemaUtil.convert(schema);
-    return new StagedSparkTable(icebergCatalog.newReplaceTableTransaction(buildIdentifier(ident), icebergSchema,
-        Spark3Util.toPartitionSpec(icebergSchema, transforms), properties.get("location"), properties,
+    return new StagedSparkTable(icebergCatalog.newReplaceTableTransaction(
+        buildIdentifier(ident),
+        icebergSchema,
+        Spark3Util.toPartitionSpec(icebergSchema, transforms),
+        properties.get("location"),
+        Spark3Util.rebuildCreateProperties(properties),
         true /* create or replace */));
   }
 


[iceberg] 03/09: Spark: Fix import for paths that include spaces (#1228)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit 505ee6108bfcd9c651d60bdc2902878a3252ea79
Author: Russell Spitzer <ru...@GMAIL.COM>
AuthorDate: Wed Jul 22 15:11:52 2020 -0500

    Spark: Fix import for paths that include spaces (#1228)
    
    In order to avoid changing the API and SparkSQL compatible types we will fix the whitespace issue by
    replacing the encoded string representation with a decoded string representation. We use a
    method identical to Apache Spark, taking the Hadoop Path representation of the URI and getting the
    string representation from that.
---
 .../main/java/org/apache/iceberg/hadoop/Util.java  | 15 ++++++
 .../org/apache/iceberg/spark/SparkTableUtil.java   |  5 +-
 .../iceberg/spark/source/TestSparkTableUtil.java   | 63 ++++++++++++++++++++++
 3 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/hadoop/Util.java b/core/src/main/java/org/apache/iceberg/hadoop/Util.java
index 24f3d6c..bdb334b 100644
--- a/core/src/main/java/org/apache/iceberg/hadoop/Util.java
+++ b/core/src/main/java/org/apache/iceberg/hadoop/Util.java
@@ -20,6 +20,7 @@
 package org.apache.iceberg.hadoop;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Set;
@@ -78,4 +79,18 @@ public class Util {
 
     return locations.toArray(HadoopInputFile.NO_LOCATION_PREFERENCE);
   }
+
+  /**
+   * From Apache Spark
+   *
+   * Convert URI to String.
+   * Since URI.toString does not decode the uri, e.g. change '%25' to '%'.
+   * Here we create a hadoop Path with the given URI, and rely on Path.toString
+   * to decode the uri
+   * @param uri the URI of the path
+   * @return the String of the path
+   */
+  public static String uriToString(URI uri) {
+    return new Path(uri).toString();
+  }
 }
diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
index 38b6fa8..828ec4f 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
@@ -49,6 +49,7 @@ import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.hadoop.HadoopFileIO;
 import org.apache.iceberg.hadoop.HadoopInputFile;
 import org.apache.iceberg.hadoop.SerializableConfiguration;
+import org.apache.iceberg.hadoop.Util;
 import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.orc.OrcMetrics;
@@ -386,7 +387,7 @@ public class SparkTableUtil {
     Preconditions.checkArgument(serde.nonEmpty() || table.provider().nonEmpty(),
         "Partition format should be defined");
 
-    String uri = String.valueOf(locationUri.get());
+    String uri = Util.uriToString(locationUri.get());
     String format = serde.nonEmpty() ? serde.get() : table.provider().get();
 
     Map<String, String> partitionSpec = JavaConverters.mapAsJavaMapConverter(partition.spec()).asJava();
@@ -495,7 +496,7 @@ public class SparkTableUtil {
       MetricsConfig metricsConfig = MetricsConfig.fromProperties(targetTable.properties());
 
       List<DataFile> files = listPartition(
-          partition, sourceTable.location().toString(), format.get(), spec, conf, metricsConfig);
+          partition, Util.uriToString(sourceTable.location()), format.get(), spec, conf, metricsConfig);
 
       AppendFiles append = targetTable.newAppend();
       files.forEach(append::appendFile);
diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java b/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
index b73c036..1cc489a 100644
--- a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
+++ b/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.spark.SparkTableUtil;
 import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
 import org.apache.iceberg.types.Types;
 import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SQLContext;
 import org.apache.spark.sql.SparkSession;
@@ -297,4 +298,66 @@ public class TestSparkTableUtil extends HiveTableBaseTest {
 
     Assert.assertEquals(expected.stream().map(SimpleRecord::getData).collect(Collectors.toList()), actual);
   }
+
+  @Test
+  public void testImportPartitionedWithWhitespace() throws Exception {
+    String partitionCol = "dAtA sPaced";
+    String spacedTableName = "whitespacetable";
+    String whiteSpaceKey = "some key value";
+
+    List<SimpleRecord> spacedRecords = Lists.newArrayList(new SimpleRecord(1, whiteSpaceKey));
+
+    File icebergLocation = temp.newFolder("partitioned_table");
+
+    spark.createDataFrame(spacedRecords, SimpleRecord.class)
+        .withColumnRenamed("data", partitionCol)
+        .write().mode("overwrite").partitionBy(partitionCol).format("parquet")
+        .saveAsTable(spacedTableName);
+
+    TableIdentifier source = spark.sessionState().sqlParser()
+        .parseTableIdentifier(spacedTableName);
+    HadoopTables tables = new HadoopTables(spark.sessionState().newHadoopConf());
+    Table table = tables.create(SparkSchemaUtil.schemaForTable(spark, spacedTableName),
+        SparkSchemaUtil.specForTable(spark, spacedTableName),
+        ImmutableMap.of(),
+        icebergLocation.getCanonicalPath());
+    File stagingDir = temp.newFolder("staging-dir");
+    SparkTableUtil.importSparkTable(spark, source, table, stagingDir.toString());
+    List<SimpleRecord> results = spark.read().format("iceberg").load(icebergLocation.toString())
+        .withColumnRenamed(partitionCol, "data")
+        .as(Encoders.bean(SimpleRecord.class))
+        .collectAsList();
+
+    Assert.assertEquals("Data should match", spacedRecords, results);
+  }
+
+  @Test
+  public void testImportUnpartitionedWithWhitespace() throws Exception {
+    String spacedTableName = "whitespacetable";
+    String whiteSpaceKey = "some key value";
+
+    List<SimpleRecord> spacedRecords = Lists.newArrayList(new SimpleRecord(1, whiteSpaceKey));
+
+    File whiteSpaceOldLocation = temp.newFolder("white space location");
+    File icebergLocation = temp.newFolder("partitioned_table");
+
+    spark.createDataFrame(spacedRecords, SimpleRecord.class)
+        .write().mode("overwrite").parquet(whiteSpaceOldLocation.getPath());
+
+    spark.catalog().createExternalTable(spacedTableName, whiteSpaceOldLocation.getPath());
+
+    TableIdentifier source = spark.sessionState().sqlParser()
+        .parseTableIdentifier(spacedTableName);
+    HadoopTables tables = new HadoopTables(spark.sessionState().newHadoopConf());
+    Table table = tables.create(SparkSchemaUtil.schemaForTable(spark, spacedTableName),
+        SparkSchemaUtil.specForTable(spark, spacedTableName),
+        ImmutableMap.of(),
+        icebergLocation.getCanonicalPath());
+    File stagingDir = temp.newFolder("staging-dir");
+    SparkTableUtil.importSparkTable(spark, source, table, stagingDir.toString());
+    List<SimpleRecord> results = spark.read().format("iceberg").load(icebergLocation.toString())
+        .as(Encoders.bean(SimpleRecord.class)).collectAsList();
+
+    Assert.assertEquals("Data should match", spacedRecords, results);
+  }
 }


[iceberg] 06/09: Implement negate in bound predicates (#1282)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit 37a5709f5206ef84bd1b35a3a052a0c3d58355a8
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Jul 31 15:06:31 2020 -0700

    Implement negate in bound predicates (#1282)
---
 .../iceberg/expressions/BoundLiteralPredicate.java |  5 ++
 .../iceberg/expressions/BoundUnaryPredicate.java   |  5 ++
 .../iceberg/expressions/TestExpressionHelpers.java | 57 ++++++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
index 698b839..e0770ce 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
@@ -32,6 +32,11 @@ public class BoundLiteralPredicate<T> extends BoundPredicate<T> {
     this.literal = lit;
   }
 
+  @Override
+  public Expression negate() {
+    return new BoundLiteralPredicate<>(op().negate(), term(), literal);
+  }
+
   public Literal<T> literal() {
     return literal;
   }
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
index c5835ba..528dd1f 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
@@ -25,6 +25,11 @@ public class BoundUnaryPredicate<T> extends BoundPredicate<T> {
   }
 
   @Override
+  public Expression negate() {
+    return new BoundUnaryPredicate<>(op().negate(), term());
+  }
+
+  @Override
   public boolean isUnaryPredicate() {
     return true;
   }
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
index 5bf336f..302e4a6 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
@@ -20,6 +20,9 @@
 package org.apache.iceberg.expressions;
 
 import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -29,11 +32,20 @@ import static org.apache.iceberg.expressions.Expressions.and;
 import static org.apache.iceberg.expressions.Expressions.bucket;
 import static org.apache.iceberg.expressions.Expressions.day;
 import static org.apache.iceberg.expressions.Expressions.equal;
+import static org.apache.iceberg.expressions.Expressions.greaterThan;
+import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
 import static org.apache.iceberg.expressions.Expressions.hour;
+import static org.apache.iceberg.expressions.Expressions.in;
+import static org.apache.iceberg.expressions.Expressions.isNull;
 import static org.apache.iceberg.expressions.Expressions.lessThan;
+import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual;
 import static org.apache.iceberg.expressions.Expressions.month;
 import static org.apache.iceberg.expressions.Expressions.not;
+import static org.apache.iceberg.expressions.Expressions.notEqual;
+import static org.apache.iceberg.expressions.Expressions.notIn;
+import static org.apache.iceberg.expressions.Expressions.notNull;
 import static org.apache.iceberg.expressions.Expressions.or;
+import static org.apache.iceberg.expressions.Expressions.rewriteNot;
 import static org.apache.iceberg.expressions.Expressions.truncate;
 import static org.apache.iceberg.expressions.Expressions.year;
 
@@ -78,6 +90,51 @@ public class TestExpressionHelpers {
   }
 
   @Test
+  public void testRewriteNot() {
+    StructType struct = StructType.of(NestedField.optional(1, "a", Types.IntegerType.get()));
+    Expression[][] expressions = new Expression[][] {
+        // (rewritten pred, original pred) pairs
+        { isNull("a"), isNull("a") },
+        { notNull("a"), not(isNull("a")) },
+        { notNull("a"), notNull("a") },
+        { isNull("a"), not(notNull("a")) },
+        { equal("a", 5), equal("a", 5) },
+        { notEqual("a", 5), not(equal("a", 5)) },
+        { notEqual("a", 5), notEqual("a", 5) },
+        { equal("a", 5), not(notEqual("a", 5)) },
+        { in("a", 5, 6), in("a", 5, 6) },
+        { notIn("a", 5, 6), not(in("a", 5, 6)) },
+        { notIn("a", 5, 6), notIn("a", 5, 6) },
+        { in("a", 5, 6), not(notIn("a", 5, 6)) },
+        { lessThan("a", 5), lessThan("a", 5) },
+        { greaterThanOrEqual("a", 5), not(lessThan("a", 5)) },
+        { greaterThanOrEqual("a", 5), greaterThanOrEqual("a", 5) },
+        { lessThan("a", 5), not(greaterThanOrEqual("a", 5)) },
+        { lessThanOrEqual("a", 5), lessThanOrEqual("a", 5) },
+        { greaterThan("a", 5), not(lessThanOrEqual("a", 5)) },
+        { greaterThan("a", 5), greaterThan("a", 5) },
+        { lessThanOrEqual("a", 5), not(greaterThan("a", 5)) },
+        { or(equal("a", 5), isNull("a")), or(equal("a", 5), isNull("a")) },
+        { and(notEqual("a", 5), notNull("a")), not(or(equal("a", 5), isNull("a"))) },
+        { and(notEqual("a", 5), notNull("a")), and(notEqual("a", 5), notNull("a")) },
+        { or(equal("a", 5), isNull("a")), not(and(notEqual("a", 5), notNull("a"))) },
+        { or(equal("a", 5), notNull("a")), or(equal("a", 5), not(isNull("a"))) },
+    };
+
+    for (Expression[] pair : expressions) {
+      // unbound rewrite
+      Assert.assertEquals(String.format("rewriteNot(%s) should be %s", pair[1], pair[0]),
+          pair[0].toString(), rewriteNot(pair[1]).toString());
+
+      // bound rewrite
+      Expression expectedBound = Binder.bind(struct, pair[0]);
+      Expression toRewriteBound = Binder.bind(struct, pair[1]);
+      Assert.assertEquals(String.format("rewriteNot(%s) should be %s", toRewriteBound, expectedBound),
+          expectedBound.toString(), rewriteNot(toRewriteBound).toString());
+    }
+  }
+
+  @Test
   public void testTransformExpressions() {
     Assert.assertEquals("Should produce the correct expression string",
         "year(ref(name=\"ts\")) == \"2019\"",


[iceberg] 04/09: Fix Guava imports (#1251)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit b012d67d184fb3b6597f94c9c184c3cc3d33d48a
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Sun Jul 26 13:48:29 2020 -0700

    Fix Guava imports (#1251)
---
 .baseline/checkstyle/checkstyle.xml                             | 2 +-
 build.gradle                                                    | 2 ++
 spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java   | 2 +-
 spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java | 4 ++--
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
index 11c8841..8a64ffb 100644
--- a/.baseline/checkstyle/checkstyle.xml
+++ b/.baseline/checkstyle/checkstyle.xml
@@ -130,7 +130,7 @@
             <message key="import.illegal" value="Must not import javafx classes because some OpenJDK builds do not include javafx."/>
         </module>
         <module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
-            <property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava"/>
+            <property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava, org.sparkproject.guava, org.glassfish.jersey.internal.guava."/>
             <message key="import.illegal" value="Must not import repackaged classes."/>
         </module>
         <module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
diff --git a/build.gradle b/build.gradle
index 063fbdd..1879e80 100644
--- a/build.gradle
+++ b/build.gradle
@@ -84,6 +84,8 @@ subprojects {
       if (project.name != 'iceberg-bundled-guava') {
         exclude group: 'com.google.guava', module: 'guava'
       }
+      // contains a copy of Guava
+      exclude group: 'org.apache.spark', module: 'spark-network-common_2.12'
     }
 
     all {
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index a41e65f..03d39be 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -37,6 +37,7 @@ import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.relocated.com.google.common.base.Joiner;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
 import org.apache.iceberg.transforms.PartitionSpecVisitor;
 import org.apache.iceberg.types.Type;
@@ -52,7 +53,6 @@ import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.types.IntegerType;
 import org.apache.spark.sql.types.LongType;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.sparkproject.guava.collect.ImmutableMap;
 
 public class Spark3Util {
 
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
index c115b84..7100906 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
@@ -39,6 +39,8 @@ import org.apache.iceberg.relocated.com.google.common.base.Splitter;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
 import org.apache.iceberg.spark.source.SparkTable;
 import org.apache.iceberg.spark.source.StagedSparkTable;
 import org.apache.spark.sql.SparkSession;
@@ -58,8 +60,6 @@ import org.apache.spark.sql.connector.catalog.TableChange.SetProperty;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.types.StructType;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.glassfish.jersey.internal.guava.Sets;
-import org.sparkproject.guava.collect.Maps;
 
 /**
  * A Spark TableCatalog implementation that wraps an Iceberg {@link Catalog}.


[iceberg] 08/09: Core: Use array instead of list in BaseCombinedScanTask to fix serialization (#1285)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit e1163458fac0c1c7749e169714e17d5e2a4af203
Author: Jungtaek Lim <ka...@gmail.com>
AuthorDate: Thu Aug 6 01:37:38 2020 +0900

    Core: Use array instead of list in BaseCombinedScanTask to fix serialization (#1285)
---
 .../org/apache/iceberg/BaseCombinedScanTask.java   |  10 +-
 .../apache/iceberg/TestScanTaskSerialization.java  | 210 +++++++++++++++++++++
 .../iceberg/TestScanTaskSerialization24.java       |  29 +--
 .../apache/iceberg/TestScanTaskSerialization3.java |  29 +--
 4 files changed, 218 insertions(+), 60 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java b/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
index 98c6943..253ddfa 100644
--- a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
+++ b/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
@@ -23,22 +23,24 @@ import java.util.Collection;
 import java.util.List;
 import org.apache.iceberg.relocated.com.google.common.base.Joiner;
 import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
 
 public class BaseCombinedScanTask implements CombinedScanTask {
-  private final List<FileScanTask> tasks;
+  private final FileScanTask[] tasks;
 
   public BaseCombinedScanTask(FileScanTask... tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
+    this.tasks = tasks;
   }
 
   public BaseCombinedScanTask(List<FileScanTask> tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
+    Preconditions.checkNotNull(tasks, "tasks cannot be null");
+    this.tasks = tasks.stream().toArray(FileScanTask[]::new);
   }
 
   @Override
   public Collection<FileScanTask> files() {
-    return tasks;
+    return ImmutableList.copyOf(tasks);
   }
 
   @Override
diff --git a/spark/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java b/spark/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java
new file mode 100644
index 0000000..f443f38
--- /dev/null
+++ b/spark/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkTestBase;
+import org.apache.iceberg.spark.source.ThreeColumnRecord;
+import org.apache.iceberg.types.Types;
+import org.apache.spark.SparkConf;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+
+public abstract class TestScanTaskSerialization extends SparkTestBase {
+
+  private static final HadoopTables TABLES = new HadoopTables(new Configuration());
+  private static final Schema SCHEMA = new Schema(
+      optional(1, "c1", Types.IntegerType.get()),
+      optional(2, "c2", Types.StringType.get()),
+      optional(3, "c3", Types.StringType.get())
+  );
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  private String tableLocation = null;
+
+  @Before
+  public void setupTableLocation() throws Exception {
+    File tableDir = temp.newFolder();
+    this.tableLocation = tableDir.toURI().toString();
+  }
+
+  @Test
+  public void testBaseCombinedScanTaskKryoSerialization() throws Exception {
+    BaseCombinedScanTask scanTask = prepareBaseCombinedScanTaskForSerDeTest();
+
+    File data = temp.newFile();
+    Assert.assertTrue(data.delete());
+    Kryo kryo = new KryoSerializer(new SparkConf()).newKryo();
+
+    try (Output out = new Output(new FileOutputStream(data))) {
+      kryo.writeClassAndObject(out, scanTask);
+    }
+
+    try (Input in = new Input(new FileInputStream(data))) {
+      Object obj = kryo.readClassAndObject(in);
+      Assert.assertTrue("Should be a BaseCombinedScanTask", obj instanceof BaseCombinedScanTask);
+      checkBaseCombinedScanTask(scanTask, (BaseCombinedScanTask) obj);
+    }
+  }
+
+  @Test
+  public void testBaseCombinedScanTaskJavaSerialization() throws Exception {
+    BaseCombinedScanTask scanTask = prepareBaseCombinedScanTaskForSerDeTest();
+
+    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+    try (ObjectOutputStream out = new ObjectOutputStream(bytes)) {
+      out.writeObject(scanTask);
+    }
+
+    try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+      Object obj = in.readObject();
+      Assert.assertTrue("Should be a BaseCombinedScanTask", obj instanceof BaseCombinedScanTask);
+      checkBaseCombinedScanTask(scanTask, (BaseCombinedScanTask) obj);
+    }
+  }
+
+  private BaseCombinedScanTask prepareBaseCombinedScanTaskForSerDeTest() {
+    PartitionSpec spec = PartitionSpec.unpartitioned();
+    Map<String, String> options = Maps.newHashMap();
+    Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+    List<ThreeColumnRecord> records1 = Lists.newArrayList(
+        new ThreeColumnRecord(1, null, "AAAA"),
+        new ThreeColumnRecord(1, "BBBBBBBBBB", "BBBB")
+    );
+    writeRecords(records1);
+
+    List<ThreeColumnRecord> records2 = Lists.newArrayList(
+        new ThreeColumnRecord(2, "CCCCCCCCCC", "CCCC"),
+        new ThreeColumnRecord(2, "DDDDDDDDDD", "DDDD")
+    );
+    writeRecords(records2);
+
+    table.refresh();
+
+    CloseableIterable<FileScanTask> tasks = table.newScan().planFiles();
+    return new BaseCombinedScanTask(Lists.newArrayList(tasks));
+  }
+
+  private void checkBaseCombinedScanTask(BaseCombinedScanTask expected, BaseCombinedScanTask actual) {
+    List<FileScanTask> expectedTasks = getFileScanTasksInFilePathOrder(expected);
+    List<FileScanTask> actualTasks = getFileScanTasksInFilePathOrder(actual);
+
+    Assert.assertEquals("The number of file scan tasks should match",
+        expectedTasks.size(), actualTasks.size());
+
+    for (int i = 0; i < expectedTasks.size(); i++) {
+      FileScanTask expectedTask = expectedTasks.get(i);
+      FileScanTask actualTask = actualTasks.get(i);
+      checkFileScanTask(expectedTask, actualTask);
+    }
+  }
+
+  private List<FileScanTask> getFileScanTasksInFilePathOrder(BaseCombinedScanTask task) {
+    return task.files().stream()
+        // use file path + start position to differentiate the tasks
+        .sorted(Comparator.comparing(o -> o.file().path().toString() + "##" + o.start()))
+        .collect(Collectors.toList());
+  }
+
+  private void checkFileScanTask(FileScanTask expected, FileScanTask actual) {
+    checkDataFile(expected.file(), actual.file());
+
+    // PartitionSpec implements its own equals method
+    Assert.assertEquals("PartitionSpec doesn't match", expected.spec(), actual.spec());
+
+    Assert.assertEquals("starting position doesn't match", expected.start(), actual.start());
+
+    Assert.assertEquals("the number of bytes to scan doesn't match", expected.start(), actual.start());
+
+    // simplify comparison on residual expression via comparing toString
+    Assert.assertEquals("Residual expression doesn't match",
+        expected.residual().toString(), actual.residual().toString());
+  }
+
+  // TODO: this is a copy of TestDataFileSerialization.checkDataFile, deduplicate
+  private void checkDataFile(DataFile expected, DataFile actual) {
+    Assert.assertEquals("Should match the serialized record path",
+        expected.path(), actual.path());
+    Assert.assertEquals("Should match the serialized record format",
+        expected.format(), actual.format());
+    Assert.assertEquals("Should match the serialized record partition",
+        expected.partition().get(0, Object.class), actual.partition().get(0, Object.class));
+    Assert.assertEquals("Should match the serialized record count",
+        expected.recordCount(), actual.recordCount());
+    Assert.assertEquals("Should match the serialized record size",
+        expected.fileSizeInBytes(), actual.fileSizeInBytes());
+    Assert.assertEquals("Should match the serialized record value counts",
+        expected.valueCounts(), actual.valueCounts());
+    Assert.assertEquals("Should match the serialized record null value counts",
+        expected.nullValueCounts(), actual.nullValueCounts());
+    Assert.assertEquals("Should match the serialized record lower bounds",
+        expected.lowerBounds(), actual.lowerBounds());
+    Assert.assertEquals("Should match the serialized record upper bounds",
+        expected.upperBounds(), actual.upperBounds());
+    Assert.assertEquals("Should match the serialized record key metadata",
+        expected.keyMetadata(), actual.keyMetadata());
+    Assert.assertEquals("Should match the serialized record offsets",
+        expected.splitOffsets(), actual.splitOffsets());
+    Assert.assertEquals("Should match the serialized record offsets",
+        expected.keyMetadata(), actual.keyMetadata());
+  }
+
+  private void writeRecords(List<ThreeColumnRecord> records) {
+    Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
+    writeDF(df);
+  }
+
+  private void writeDF(Dataset<Row> df) {
+    df.select("c1", "c2", "c3")
+        .write()
+        .format("iceberg")
+        .mode("append")
+        .save(tableLocation);
+  }
+}
diff --git a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java b/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java
similarity index 50%
copy from core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
copy to spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java
index 98c6943..28184b7 100644
--- a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
+++ b/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java
@@ -19,32 +19,5 @@
 
 package org.apache.iceberg;
 
-import java.util.Collection;
-import java.util.List;
-import org.apache.iceberg.relocated.com.google.common.base.Joiner;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-
-public class BaseCombinedScanTask implements CombinedScanTask {
-  private final List<FileScanTask> tasks;
-
-  public BaseCombinedScanTask(FileScanTask... tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
-  }
-
-  public BaseCombinedScanTask(List<FileScanTask> tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
-  }
-
-  @Override
-  public Collection<FileScanTask> files() {
-    return tasks;
-  }
-
-  @Override
-  public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("tasks", Joiner.on(", ").join(tasks))
-        .toString();
-  }
+public class TestScanTaskSerialization24 extends TestScanTaskSerialization {
 }
diff --git a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java b/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java
similarity index 50%
copy from core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
copy to spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java
index 98c6943..035aa49 100644
--- a/core/src/main/java/org/apache/iceberg/BaseCombinedScanTask.java
+++ b/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java
@@ -19,32 +19,5 @@
 
 package org.apache.iceberg;
 
-import java.util.Collection;
-import java.util.List;
-import org.apache.iceberg.relocated.com.google.common.base.Joiner;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-
-public class BaseCombinedScanTask implements CombinedScanTask {
-  private final List<FileScanTask> tasks;
-
-  public BaseCombinedScanTask(FileScanTask... tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
-  }
-
-  public BaseCombinedScanTask(List<FileScanTask> tasks) {
-    this.tasks = ImmutableList.copyOf(tasks);
-  }
-
-  @Override
-  public Collection<FileScanTask> files() {
-    return tasks;
-  }
-
-  @Override
-  public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("tasks", Joiner.on(", ").join(tasks))
-        .toString();
-  }
+public class TestScanTaskSerialization3 extends TestScanTaskSerialization {
 }


[iceberg] 05/09: Ban imports from org.apache.commons.compress.utils (#1278)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit 3f68359dce949b99940a030a85aa67517f1d97ac
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Jul 31 14:11:08 2020 -0700

    Ban imports from org.apache.commons.compress.utils (#1278)
    
    Co-authored-by: Chen, Junjie <ch...@gmail.com>
---
 .baseline/checkstyle/checkstyle.xml                               | 2 +-
 core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
index 8a64ffb..1a2155e 100644
--- a/.baseline/checkstyle/checkstyle.xml
+++ b/.baseline/checkstyle/checkstyle.xml
@@ -130,7 +130,7 @@
             <message key="import.illegal" value="Must not import javafx classes because some OpenJDK builds do not include javafx."/>
         </module>
         <module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
-            <property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava, org.sparkproject.guava, org.glassfish.jersey.internal.guava."/>
+            <property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava, org.sparkproject.guava, org.glassfish.jersey.internal.guava, org.apache.commons.compress.utils"/>
             <message key="import.illegal" value="Must not import repackaged classes."/>
         </module>
         <module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java
index e08578c..39a6d82 100644
--- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java
+++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java
@@ -23,7 +23,6 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
-import org.apache.commons.compress.utils.Lists;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
@@ -34,6 +33,7 @@ import org.apache.iceberg.exceptions.RuntimeIOException;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.SeekableInputStream;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 
 /**
  * {@link InputFile} implementation using the Hadoop {@link FileSystem} API.


[iceberg] 09/09: ORC: Fix decimal and timestamp bugs (#1271)

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit 768dc08a2ce558ea309215fa1c864ec0c48a4e72
Author: openinx <op...@gmail.com>
AuthorDate: Fri Aug 7 23:58:24 2020 +0800

    ORC: Fix decimal and timestamp bugs (#1271)
---
 build.gradle                                       |   1 +
 .../apache/iceberg/data/orc/GenericOrcWriter.java  |  14 +-
 .../apache/iceberg/spark/data/SparkOrcReader.java  |   7 +-
 .../iceberg/spark/data/SparkOrcValueReaders.java   |  32 ++++-
 .../apache/iceberg/spark/data/SparkOrcWriter.java  |  16 ++-
 .../apache/iceberg/spark/data/GenericsHelpers.java |   2 +-
 .../spark/data/TestSparkRecordOrcReaderWriter.java | 148 +++++++++++++++++++++
 7 files changed, 199 insertions(+), 21 deletions(-)

diff --git a/build.gradle b/build.gradle
index 1879e80..4e8de83 100644
--- a/build.gradle
+++ b/build.gradle
@@ -451,6 +451,7 @@ project(':iceberg-spark') {
     }
     testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts')
     testCompile project(path: ':iceberg-api', configuration: 'testArtifacts')
+    testCompile project(path: ':iceberg-data', configuration: 'testArtifacts')
   }
 
   test {
diff --git a/data/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriter.java b/data/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriter.java
index 1bf466d..0bc5b90 100644
--- a/data/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriter.java
+++ b/data/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriter.java
@@ -363,8 +363,10 @@ public class GenericOrcWriter implements OrcValueWriter<Record> {
         output.isNull[rowId] = false;
         TimestampColumnVector cv = (TimestampColumnVector) output;
         cv.setIsUTC(true);
-        cv.time[rowId] = data.toInstant(ZoneOffset.UTC).toEpochMilli(); // millis
-        cv.nanos[rowId] = (data.getNano() / 1_000) * 1_000; // truncate nanos to only keep microsecond precision
+        // millis
+        cv.time[rowId] = data.toInstant(ZoneOffset.UTC).toEpochMilli();
+        // truncate nanos to only keep microsecond precision
+        cv.nanos[rowId] = data.getNano() / 1_000 * 1_000;
       }
     }
   }
@@ -578,7 +580,13 @@ public class GenericOrcWriter implements OrcValueWriter<Record> {
       case VARCHAR:
         return new StringConverter();
       case DECIMAL:
-        return schema.getPrecision() <= 18 ? new Decimal18Converter(schema) : new Decimal38Converter(schema);
+        int precision = schema.getPrecision();
+        if (precision <= 18) {
+          return new Decimal18Converter(schema);
+        } else if (precision <= 38) {
+          return new Decimal38Converter(schema);
+        }
+        throw new IllegalArgumentException("Invalid precision: " + precision);
       case TIMESTAMP:
         return new TimestampConverter();
       case TIMESTAMP_INSTANT:
diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java
index da1ee6e..6e6eb56 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java
@@ -32,7 +32,6 @@ import org.apache.orc.TypeDescription;
 import org.apache.orc.storage.ql.exec.vector.StructColumnVector;
 import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
 import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.Decimal;
 
 /**
  * Converts the OrcIterator, which returns ORC's VectorizedRowBatch to a
@@ -103,11 +102,7 @@ public class SparkOrcReader implements OrcRowReader<InternalRow> {
         case TIMESTAMP_INSTANT:
           return SparkOrcValueReaders.timestampTzs();
         case DECIMAL:
-          if (primitive.getPrecision() <= Decimal.MAX_LONG_DIGITS()) {
-            return new SparkOrcValueReaders.Decimal18Reader(primitive.getPrecision(), primitive.getScale());
-          } else {
-            return new SparkOrcValueReaders.Decimal38Reader(primitive.getPrecision(), primitive.getScale());
-          }
+          return SparkOrcValueReaders.decimals(primitive.getPrecision(), primitive.getScale());
         case CHAR:
         case VARCHAR:
         case STRING:
diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java
index 55e3117..94bb0cb 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.orc.OrcValueReader;
 import org.apache.iceberg.orc.OrcValueReaders;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.iceberg.types.Types;
 import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
@@ -55,6 +56,16 @@ class SparkOrcValueReaders {
     return TimestampTzReader.INSTANCE;
   }
 
+  public static OrcValueReader<Decimal> decimals(int precision, int scale) {
+    if (precision <= Decimal.MAX_LONG_DIGITS()) {
+      return new Decimal18Reader(precision, scale);
+    } else if (precision <= 38) {
+      return new Decimal38Reader(precision, scale);
+    } else {
+      throw new IllegalArgumentException("Invalid precision: " + precision);
+    }
+  }
+
   static OrcValueReader<?> struct(
       List<OrcValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
     return new StructReader(readers, struct, idToConstant);
@@ -159,13 +170,12 @@ class SparkOrcValueReaders {
 
     @Override
     public Long nonNullRead(ColumnVector vector, int row) {
-      TimestampColumnVector timestampVector = (TimestampColumnVector) vector;
-      return (timestampVector.time[row] / 1000) * 1_000_000 + timestampVector.nanos[row] / 1000;
+      TimestampColumnVector tcv = (TimestampColumnVector) vector;
+      return (Math.floorDiv(tcv.time[row], 1_000)) * 1_000_000 + Math.floorDiv(tcv.nanos[row], 1000);
     }
   }
 
-  static class Decimal18Reader implements OrcValueReader<Decimal> {
-    //TODO: these are being unused. check for bug
+  private static class Decimal18Reader implements OrcValueReader<Decimal> {
     private final int precision;
     private final int scale;
 
@@ -177,7 +187,15 @@ class SparkOrcValueReaders {
     @Override
     public Decimal nonNullRead(ColumnVector vector, int row) {
       HiveDecimalWritable value = ((DecimalColumnVector) vector).vector[row];
-      return new Decimal().set(value.serialize64(value.scale()), value.precision(), value.scale());
+
+      // The scale of decimal read from hive ORC file may be not equals to the expected scale. For data type
+      // decimal(10,3) and the value 10.100, the hive ORC writer will remove its trailing zero and store it
+      // as 101*10^(-1), its scale will adjust from 3 to 1. So here we could not assert that value.scale() == scale.
+      // we also need to convert the hive orc decimal to a decimal with expected precision and scale.
+      Preconditions.checkArgument(value.precision() <= precision,
+          "Cannot read value as decimal(%s,%s), too large: %s", precision, scale, value);
+
+      return new Decimal().set(value.serialize64(scale), precision, scale);
     }
   }
 
@@ -194,6 +212,10 @@ class SparkOrcValueReaders {
     public Decimal nonNullRead(ColumnVector vector, int row) {
       BigDecimal value = ((DecimalColumnVector) vector).vector[row]
           .getHiveDecimal().bigDecimalValue();
+
+      Preconditions.checkArgument(value.precision() <= precision,
+          "Cannot read value as decimal(%s,%s), too large: %s", precision, scale, value);
+
       return new Decimal().set(new scala.math.BigDecimal(value), precision, scale);
     }
   }
diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java
index c27f958..9234c98 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java
@@ -214,9 +214,9 @@ public class SparkOrcWriter implements OrcValueWriter<InternalRow> {
       } else {
         output.isNull[rowId] = false;
         TimestampColumnVector cv = (TimestampColumnVector) output;
-        long micros = data.getLong(column);
-        cv.time[rowId] = micros / 1_000; // millis
-        cv.nanos[rowId] = (int) (micros % 1_000_000) * 1_000; // nanos
+        long micros = data.getLong(column); // it could be negative
+        cv.time[rowId] = Math.floorDiv(micros, 1_000); // millis
+        cv.nanos[rowId] = (int) Math.floorMod(micros, 1_000_000) * 1_000; // nanos
       }
     }
   }
@@ -388,9 +388,13 @@ public class SparkOrcWriter implements OrcValueWriter<InternalRow> {
       case VARCHAR:
         return new StringConverter();
       case DECIMAL:
-        return schema.getPrecision() <= 18 ?
-            new Decimal18Converter(schema) :
-            new Decimal38Converter(schema);
+        int precision = schema.getPrecision();
+        if (precision <= 18) {
+          return new Decimal18Converter(schema);
+        } else if (precision <= 38) {
+          return new Decimal38Converter(schema);
+        }
+        throw new IllegalArgumentException("Invalid precision: " + precision);
       case TIMESTAMP_INSTANT:
         return new TimestampTzConverter();
       case STRUCT:
diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java b/spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java
index 821f5bd..0c4598a 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java
@@ -237,7 +237,7 @@ public class GenericsHelpers {
         break;
       case DATE:
         Assert.assertTrue("Should expect a LocalDate", expected instanceof LocalDate);
-        long expectedDays = ChronoUnit.DAYS.between(EPOCH_DAY, (LocalDate) expected);
+        int expectedDays = (int) ChronoUnit.DAYS.between(EPOCH_DAY, (LocalDate) expected);
         Assert.assertEquals("Primitive value should be equal to expected", expectedDays, actual);
         break;
       case TIMESTAMP:
diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java b/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java
new file mode 100644
index 0000000..1e7430d
--- /dev/null
+++ b/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.data;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.RandomGenericData;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.orc.GenericOrcWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.FileAppender;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+public class TestSparkRecordOrcReaderWriter extends AvroDataTest {
+  private static final int NUM_RECORDS = 200;
+
+  private void writeAndValidate(Schema schema, List<Record> expectedRecords) throws IOException {
+    final File originalFile = temp.newFile();
+    Assert.assertTrue("Delete should succeed", originalFile.delete());
+
+    // Write few generic records into the original test file.
+    try (FileAppender<Record> writer = ORC.write(Files.localOutput(originalFile))
+        .createWriterFunc(GenericOrcWriter::buildWriter)
+        .schema(schema)
+        .build()) {
+      writer.addAll(expectedRecords);
+    }
+
+    // Read into spark InternalRow from the original test file.
+    List<InternalRow> internalRows = Lists.newArrayList();
+    try (CloseableIterable<InternalRow> reader = ORC.read(Files.localInput(originalFile))
+        .project(schema)
+        .createReaderFunc(readOrcSchema -> new SparkOrcReader(schema, readOrcSchema))
+        .build()) {
+      reader.forEach(internalRows::add);
+      assertEqualsUnsafe(schema.asStruct(), expectedRecords, reader, expectedRecords.size());
+    }
+
+    final File anotherFile = temp.newFile();
+    Assert.assertTrue("Delete should succeed", anotherFile.delete());
+
+    // Write those spark InternalRows into a new file again.
+    try (FileAppender<InternalRow> writer = ORC.write(Files.localOutput(anotherFile))
+        .createWriterFunc(SparkOrcWriter::new)
+        .schema(schema)
+        .build()) {
+      writer.addAll(internalRows);
+    }
+
+    // Check whether the InternalRows are expected records.
+    try (CloseableIterable<InternalRow> reader = ORC.read(Files.localInput(anotherFile))
+        .project(schema)
+        .createReaderFunc(readOrcSchema -> new SparkOrcReader(schema, readOrcSchema))
+        .build()) {
+      assertEqualsUnsafe(schema.asStruct(), expectedRecords, reader, expectedRecords.size());
+    }
+
+    // Read into iceberg GenericRecord and check again.
+    try (CloseableIterable<Record> reader = ORC.read(Files.localInput(anotherFile))
+        .createReaderFunc(typeDesc -> GenericOrcReader.buildReader(schema, typeDesc))
+        .project(schema)
+        .build()) {
+      assertRecordEquals(expectedRecords, reader, expectedRecords.size());
+    }
+  }
+
+  @Override
+  protected void writeAndValidate(Schema schema) throws IOException {
+    List<Record> expectedRecords = RandomGenericData.generate(schema, NUM_RECORDS, 1992L);
+    writeAndValidate(schema, expectedRecords);
+  }
+
+  @Test
+  public void testDecimalWithTrailingZero() throws IOException {
+    Schema schema = new Schema(
+        required(1, "d1", Types.DecimalType.of(10, 2)),
+        required(2, "d2", Types.DecimalType.of(20, 5)),
+        required(3, "d3", Types.DecimalType.of(38, 20))
+    );
+
+    List<Record> expected = Lists.newArrayList();
+
+    GenericRecord record = GenericRecord.create(schema);
+    record.set(0, new BigDecimal("101.00"));
+    record.set(1, new BigDecimal("10.00E-3"));
+    record.set(2, new BigDecimal("1001.0000E-16"));
+
+    expected.add(record.copy());
+
+    writeAndValidate(schema, expected);
+  }
+
+  private static void assertRecordEquals(Iterable<Record> expected, Iterable<Record> actual, int size) {
+    Iterator<Record> expectedIter = expected.iterator();
+    Iterator<Record> actualIter = actual.iterator();
+    for (int i = 0; i < size; i += 1) {
+      Assert.assertTrue("Expected iterator should have more rows", expectedIter.hasNext());
+      Assert.assertTrue("Actual iterator should have more rows", actualIter.hasNext());
+      Assert.assertEquals("Should have same rows.", expectedIter.next(), actualIter.next());
+    }
+    Assert.assertFalse("Expected iterator should not have any extra rows.", expectedIter.hasNext());
+    Assert.assertFalse("Actual iterator should not have any extra rows.", actualIter.hasNext());
+  }
+
+  private static void assertEqualsUnsafe(Types.StructType struct, Iterable<Record> expected,
+                                         Iterable<InternalRow> actual, int size) {
+    Iterator<Record> expectedIter = expected.iterator();
+    Iterator<InternalRow> actualIter = actual.iterator();
+    for (int i = 0; i < size; i += 1) {
+      Assert.assertTrue("Expected iterator should have more rows", expectedIter.hasNext());
+      Assert.assertTrue("Actual iterator should have more rows", actualIter.hasNext());
+      GenericsHelpers.assertEqualsUnsafe(struct, expectedIter.next(), actualIter.next());
+    }
+    Assert.assertFalse("Expected iterator should not have any extra rows.", expectedIter.hasNext());
+    Assert.assertFalse("Actual iterator should not have any extra rows.", actualIter.hasNext());
+  }
+}