You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@orc.apache.org by GitBox <gi...@apache.org> on 2020/08/31 21:11:41 UTC

[GitHub] [orc] omalley opened a new pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

omalley opened a new pull request #542:
URL: https://github.com/apache/orc/pull/542


   This fixes evaluation of predicate push down filters for columns of ints and doubles with all null values.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] shardulm94 commented on a change in pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
shardulm94 commented on a change in pull request #542:
URL: https://github.com/apache/orc/pull/542#discussion_r480406095



##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .isNull("int1", PredicateLeaf.Type.LONG)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(3500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+  }
+
+  @Test
+  public void testPredicatePushdownIssue2() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      if (i % 2 == 0) {
+        batch.cols[0].noNulls = false;
+        batch.cols[0].isNull[i] = true; // every other value is null or 1
+        batch.cols[1].noNulls = false;
+        batch.cols[1].isNull[i] = true; // every other value is null or "val"
+      } else {
+        batch.cols[0].isNull[i] = false;
+        ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+        batch.cols[1].isNull[i] = false;
+        ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+      }
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // values for int1 are either null or 1, so not(in(int1, 1)) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned

Review comment:
       This should now change from `should always be true` to `should always be false` and from `hence all row groups should be returned` to `hence no row groups should be returned` as my initial understanding when I wrote the comment was wrong

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .isNull("int1", PredicateLeaf.Type.LONG)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(3500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+  }
+
+  @Test
+  public void testPredicatePushdownIssue2() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      if (i % 2 == 0) {
+        batch.cols[0].noNulls = false;
+        batch.cols[0].isNull[i] = true; // every other value is null or 1
+        batch.cols[1].noNulls = false;
+        batch.cols[1].isNull[i] = true; // every other value is null or "val"
+      } else {
+        batch.cols[0].isNull[i] = false;
+        ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+        batch.cols[1].isNull[i] = false;
+        ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+      }
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // values for int1 are either null or 1, so not(in(int1, 1)) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .in("int1", PredicateLeaf.Type.LONG, 1L)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(2500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+
+    // values for string1 are either null or "val", so not(in(string1, "val")) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned

Review comment:
       This should now change from `should always be true` to `should always be false` and from `hence all row groups should be returned` to `hence no row groups should be returned` as my initial understanding when I wrote the comment was wrong




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] omalley commented on pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
omalley commented on pull request #542:
URL: https://github.com/apache/orc/pull/542#issuecomment-684109875


   Ok, I just made some additional changes to test more variants. @shardulm94 let me know what you think.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] shardulm94 commented on a change in pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
shardulm94 commented on a change in pull request #542:
URL: https://github.com/apache/orc/pull/542#discussion_r480405386



##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {

Review comment:
       Nit: Make method name more descriptive?

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .isNull("int1", PredicateLeaf.Type.LONG)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(3500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+  }
+
+  @Test
+  public void testPredicatePushdownIssue2() throws Exception {

Review comment:
       Nit: Make method name more descriptive?

##########
File path: java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
##########
@@ -403,16 +406,18 @@ public void testGetMin() throws Exception {
 
   private static OrcProto.ColumnStatistics createIntStats(Long min,
                                                           Long max) {
+    OrcProto.ColumnStatistics.Builder result =
+        OrcProto.ColumnStatistics.newBuilder();
     OrcProto.IntegerStatistics.Builder intStats =
         OrcProto.IntegerStatistics.newBuilder();
     if (min != null) {
       intStats.setMinimum(min);
+      result.setNumberOfValues(1);

Review comment:
       I am not sure why this is needed? Also if min != max wouldn't we have > 1 values?
   PSA: I am not familiar with the Orc codebase, especially the protobuf parts. So may be totally missing the point.

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .isNull("int1", PredicateLeaf.Type.LONG)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(3500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+  }
+
+  @Test
+  public void testPredicatePushdownIssue2() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      if (i % 2 == 0) {
+        batch.cols[0].noNulls = false;
+        batch.cols[0].isNull[i] = true; // every other value is null or 1
+        batch.cols[1].noNulls = false;
+        batch.cols[1].isNull[i] = true; // every other value is null or "val"
+      } else {
+        batch.cols[0].isNull[i] = false;
+        ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+        batch.cols[1].isNull[i] = false;
+        ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+      }
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // values for int1 are either null or 1, so not(in(int1, 1)) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned

Review comment:
       This should now change from `hence all row groups should be returned` to `hence no row groups should be returned` as my initial understanding when I wrote the comment was wrong

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read

Review comment:
       Typo `not rows should be read` -> `no rows should be read`

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,115 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  @Test
+  public void testPredicatePushdownIssue1() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[i] = true; // all nulls for int1
+      ((BytesColumnVector) batch.cols[1]).setVal(i,
+          Integer.toHexString(10*i).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // all values for int1 are null, so not(isNull(int1)) should always be false and not rows should be read
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .isNull("int1", PredicateLeaf.Type.LONG)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(3500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+  }
+
+  @Test
+  public void testPredicatePushdownIssue2() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf)
+            .setSchema(schema)
+            .stripeSize(400000L)
+            .compress(CompressionKind.NONE)
+            .bufferSize(500)
+            .rowIndexStride(1000)
+            .version(fileFormat));
+    VectorizedRowBatch batch = schema.createRowBatch();
+    batch.ensureSize(3500);
+    batch.size = 3500;
+    for(int i=0; i < 3500; ++i) {
+      if (i % 2 == 0) {
+        batch.cols[0].noNulls = false;
+        batch.cols[0].isNull[i] = true; // every other value is null or 1
+        batch.cols[1].noNulls = false;
+        batch.cols[1].isNull[i] = true; // every other value is null or "val"
+      } else {
+        batch.cols[0].isNull[i] = false;
+        ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+        batch.cols[1].isNull[i] = false;
+        ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+      }
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+    Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs));
+    assertEquals(3500, reader.getNumberOfRows());
+
+    // values for int1 are either null or 1, so not(in(int1, 1)) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned
+    SearchArgument sarg = SearchArgumentFactory.newBuilder()
+        .startNot()
+        .in("int1", PredicateLeaf.Type.LONG, 1L)
+        .end()
+        .build();
+
+    RecordReader rows = reader.rows(reader.options()
+        .range(0L, Long.MAX_VALUE)
+        .searchArgument(sarg, new String[]{}));
+    batch = reader.getSchema().createRowBatch(2500);
+
+    rows.nextBatch(batch);
+    assertEquals(0, batch.size);
+
+    // values for string1 are either null or "val", so not(in(string1, "val")) should always be true because null
+    // occurs every 2nd row and hence all row groups should be returned

Review comment:
       This should now change from `hence all row groups should be returned` to `hence no row groups should be returned` as my initial understanding when I wrote the comment was wrong




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] omalley closed pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
omalley closed pull request #542:
URL: https://github.com/apache/orc/pull/542


   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] wgtmac commented on a change in pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
wgtmac commented on a change in pull request #542:
URL: https://github.com/apache/orc/pull/542#discussion_r480748085



##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,137 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  /**
+   * Test predicate pushdown on nulls, with different combinations of
+   * values and nulls.
+   */
+  @Test
+  public void testPredicatePushdownAllNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf).setSchema(schema).rowIndexStride(1024).version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.size = 1024;
+
+      // write 1024 rows of (null, "val")
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[0] = true;
+      batch.cols[0].isRepeating = true;
+      batch.cols[1].isRepeating = true;
+      ((BytesColumnVector) batch.cols[1]).setVal(0, "val".getBytes(StandardCharsets.UTF_8));
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (1, null)
+      batch.cols[0].isNull[0] = false;
+      ((LongColumnVector) batch.cols[0]).vector[0] = 123;
+      batch.cols[1].noNulls = false;
+      batch.cols[1].isNull[0] = true;
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (null, null)
+      batch.cols[0].isNull[0] = true;
+      writer.addRowBatch(batch);
+    }
+
+    try (Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs))) {
+      assertEquals(3072, reader.getNumberOfRows());
+      VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+
+      // int1 is not null
+      SearchArgument sarg =
+          SearchArgumentFactory.newBuilder()
+              .startNot()
+              .isNull("int1", PredicateLeaf.Type.LONG)
+              .end()
+              .build();
+      // should find one row group
+      try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, new String[]{}))) {
+        rows.nextBatch(batch);
+        assertEquals(1024, batch.size);
+        assertEquals(true, batch.cols[0].isRepeating);
+        assertEquals(123, ((LongColumnVector) batch.cols[0]).vector[0]);
+        assertEquals(false, rows.nextBatch(batch));
+      }
+
+      // string1 is not null
+      sarg = SearchArgumentFactory.newBuilder()
+          .startNot()
+          .isNull("string1", PredicateLeaf.Type.STRING)
+          .end()
+          .build();
+      // should find one row group
+      try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, new String[]{}))) {
+        rows.nextBatch(batch);
+        assertEquals(1024, batch.size);
+        assertEquals(true, batch.cols[1].isRepeating);
+        assertEquals("val", ((BytesColumnVector) batch.cols[1]).toString(0));
+        assertEquals(false, rows.nextBatch(batch));
+      }
+    }
+  }
+
+  /**
+   * Write three row groups, one with (null, null), one with (1, "val"), and one with
+   * alternating rows.
+   */
+  @Test
+  public void testPredicatePushdownMixedNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+                           OrcFile.writerOptions(conf)
+                                  .setSchema(schema)
+                                  .rowIndexStride(1024)
+                                  .version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.cols[0].noNulls = false;
+      batch.cols[1].noNulls = false;
+      batch.size = 1024;
+      for (int b = 0; b < 3; ++b) {
+        for (int i = 0; i < batch.size; ++i) {
+          if (b == 0 || (b == 2 && i % 2 == 0)) {
+            batch.cols[0].isNull[i] = true; // every other value is null or 1
+            batch.cols[1].isNull[i] = true; // every other value is null or "val"
+          } else {
+            batch.cols[0].isNull[i] = false;
+            ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+            batch.cols[1].isNull[i] = false;
+            ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+          }
+        }
+        writer.addRowBatch(batch);
+      }
+    }
+
+    try (Reader reader = OrcFile.createReader(testFilePath,
+                          OrcFile.readerOptions(conf).filesystem(fs))) {
+      assertEquals(3*1024, reader.getNumberOfRows());
+      VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+
+      // int1 is not null -- should select 0 of the row groups

Review comment:
       should be consistent with sarg, same for line 4183

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,137 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  /**
+   * Test predicate pushdown on nulls, with different combinations of
+   * values and nulls.
+   */
+  @Test
+  public void testPredicatePushdownAllNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf).setSchema(schema).rowIndexStride(1024).version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.size = 1024;
+
+      // write 1024 rows of (null, "val")
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[0] = true;
+      batch.cols[0].isRepeating = true;
+      batch.cols[1].isRepeating = true;
+      ((BytesColumnVector) batch.cols[1]).setVal(0, "val".getBytes(StandardCharsets.UTF_8));
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (1, null)

Review comment:
       (123, null)




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] omalley commented on a change in pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
omalley commented on a change in pull request #542:
URL: https://github.com/apache/orc/pull/542#discussion_r480436966



##########
File path: java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
##########
@@ -403,16 +406,18 @@ public void testGetMin() throws Exception {
 
   private static OrcProto.ColumnStatistics createIntStats(Long min,
                                                           Long max) {
+    OrcProto.ColumnStatistics.Builder result =
+        OrcProto.ColumnStatistics.newBuilder();
     OrcProto.IntegerStatistics.Builder intStats =
         OrcProto.IntegerStatistics.newBuilder();
     if (min != null) {
       intStats.setMinimum(min);
+      result.setNumberOfValues(1);

Review comment:
       Ok, the fix was about handling ColumnStatistics when there were no non-null values. So the added branch checks the number of values in the statistics. In real files, the only way to have a minimum or maximum is to have had at least one value, so it works.
   
   In the sarg tests though, we were creating artificial ColumnStatistics that had minimum and maximum without any values. They broke with the new code. The fix was to make the artificial ColumnStatistics have a non-zero number of values.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] shardulm94 commented on pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
shardulm94 commented on pull request #542:
URL: https://github.com/apache/orc/pull/542#issuecomment-684151474


   LGTM! 👍 


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [orc] omalley commented on a change in pull request #542: ORC-623 Fix evaluation of SArgs over all null columns for ints and doubles.

Posted by GitBox <gi...@apache.org>.
omalley commented on a change in pull request #542:
URL: https://github.com/apache/orc/pull/542#discussion_r481278167



##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,137 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  /**
+   * Test predicate pushdown on nulls, with different combinations of
+   * values and nulls.
+   */
+  @Test
+  public void testPredicatePushdownAllNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf).setSchema(schema).rowIndexStride(1024).version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.size = 1024;
+
+      // write 1024 rows of (null, "val")
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[0] = true;
+      batch.cols[0].isRepeating = true;
+      batch.cols[1].isRepeating = true;
+      ((BytesColumnVector) batch.cols[1]).setVal(0, "val".getBytes(StandardCharsets.UTF_8));
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (1, null)
+      batch.cols[0].isNull[0] = false;
+      ((LongColumnVector) batch.cols[0]).vector[0] = 123;
+      batch.cols[1].noNulls = false;
+      batch.cols[1].isNull[0] = true;
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (null, null)
+      batch.cols[0].isNull[0] = true;
+      writer.addRowBatch(batch);
+    }
+
+    try (Reader reader = OrcFile.createReader(testFilePath,
+        OrcFile.readerOptions(conf).filesystem(fs))) {
+      assertEquals(3072, reader.getNumberOfRows());
+      VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+
+      // int1 is not null
+      SearchArgument sarg =
+          SearchArgumentFactory.newBuilder()
+              .startNot()
+              .isNull("int1", PredicateLeaf.Type.LONG)
+              .end()
+              .build();
+      // should find one row group
+      try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, new String[]{}))) {
+        rows.nextBatch(batch);
+        assertEquals(1024, batch.size);
+        assertEquals(true, batch.cols[0].isRepeating);
+        assertEquals(123, ((LongColumnVector) batch.cols[0]).vector[0]);
+        assertEquals(false, rows.nextBatch(batch));
+      }
+
+      // string1 is not null
+      sarg = SearchArgumentFactory.newBuilder()
+          .startNot()
+          .isNull("string1", PredicateLeaf.Type.STRING)
+          .end()
+          .build();
+      // should find one row group
+      try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, new String[]{}))) {
+        rows.nextBatch(batch);
+        assertEquals(1024, batch.size);
+        assertEquals(true, batch.cols[1].isRepeating);
+        assertEquals("val", ((BytesColumnVector) batch.cols[1]).toString(0));
+        assertEquals(false, rows.nextBatch(batch));
+      }
+    }
+  }
+
+  /**
+   * Write three row groups, one with (null, null), one with (1, "val"), and one with
+   * alternating rows.
+   */
+  @Test
+  public void testPredicatePushdownMixedNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+                           OrcFile.writerOptions(conf)
+                                  .setSchema(schema)
+                                  .rowIndexStride(1024)
+                                  .version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.cols[0].noNulls = false;
+      batch.cols[1].noNulls = false;
+      batch.size = 1024;
+      for (int b = 0; b < 3; ++b) {
+        for (int i = 0; i < batch.size; ++i) {
+          if (b == 0 || (b == 2 && i % 2 == 0)) {
+            batch.cols[0].isNull[i] = true; // every other value is null or 1
+            batch.cols[1].isNull[i] = true; // every other value is null or "val"
+          } else {
+            batch.cols[0].isNull[i] = false;
+            ((LongColumnVector) batch.cols[0]).vector[i] = 1;
+            batch.cols[1].isNull[i] = false;
+            ((BytesColumnVector) batch.cols[1]).setVal(i, "val".getBytes(StandardCharsets.UTF_8));
+          }
+        }
+        writer.addRowBatch(batch);
+      }
+    }
+
+    try (Reader reader = OrcFile.createReader(testFilePath,
+                          OrcFile.readerOptions(conf).filesystem(fs))) {
+      assertEquals(3*1024, reader.getNumberOfRows());
+      VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+
+      // int1 is not null -- should select 0 of the row groups

Review comment:
       Thanks for the catch, @wgtmac !

##########
File path: java/core/src/test/org/apache/orc/TestVectorOrcFile.java
##########
@@ -4061,6 +4061,137 @@ public void testPredicatePushdownWithNan() throws Exception {
     assertEquals(0, batch.size);
   }
 
+  /**
+   * Test predicate pushdown on nulls, with different combinations of
+   * values and nulls.
+   */
+  @Test
+  public void testPredicatePushdownAllNulls() throws Exception {
+    TypeDescription schema = createInnerSchema();
+    try (Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf).setSchema(schema).rowIndexStride(1024).version(fileFormat))) {
+      VectorizedRowBatch batch = schema.createRowBatch();
+      batch.size = 1024;
+
+      // write 1024 rows of (null, "val")
+      batch.cols[0].noNulls = false;
+      batch.cols[0].isNull[0] = true;
+      batch.cols[0].isRepeating = true;
+      batch.cols[1].isRepeating = true;
+      ((BytesColumnVector) batch.cols[1]).setVal(0, "val".getBytes(StandardCharsets.UTF_8));
+      writer.addRowBatch(batch);
+
+      // write 1024 rows of (1, null)

Review comment:
       oops.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org