You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2022/12/30 16:55:20 UTC
[orc] branch main updated: ORC-1339: Remove orc.sarg.to.filter default value assumption in test cases
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 7a186c4bf ORC-1339: Remove orc.sarg.to.filter default value assumption in test cases
7a186c4bf is described below
commit 7a186c4bf5381ede0f94e5e7d34b1a24a03540ac
Author: William Hyun <wi...@apache.org>
AuthorDate: Fri Dec 30 08:55:09 2022 -0800
ORC-1339: Remove orc.sarg.to.filter default value assumption in test cases
### What changes were proposed in this pull request?
This PR aims to remove `orc.sarg.to.filter` default value assumption in test cases.
### Why are the changes needed?
To improve test cases.
### How was this patch tested?
Pass the CIs with updated test cases.
Closes #1350 from williamhyun/sarg.
Authored-by: William Hyun <wi...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../core/src/test/org/apache/orc/TestRowFilteringIOSkip.java | 2 ++
java/core/src/test/org/apache/orc/TestStringDictionary.java | 6 ++++--
java/core/src/test/org/apache/orc/TestVectorOrcFile.java | 12 ++++++++----
.../src/test/org/apache/orc/mapred/TestOrcFileEvolution.java | 2 +-
.../apache/orc/mapreduce/TestMapreduceOrcOutputFormat.java | 1 +
5 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/java/core/src/test/org/apache/orc/TestRowFilteringIOSkip.java b/java/core/src/test/org/apache/orc/TestRowFilteringIOSkip.java
index 863bb7da4..bffa76e5c 100644
--- a/java/core/src/test/org/apache/orc/TestRowFilteringIOSkip.java
+++ b/java/core/src/test/org/apache/orc/TestRowFilteringIOSkip.java
@@ -254,6 +254,7 @@ public class TestRowFilteringIOSkip {
.in("f1", PredicateLeaf.Type.LONG, 0L)
.build();
Reader.Options options = r.options()
+ .allowSARGToFilter(false)
.useSelected(true)
.searchArgument(sarg, new String[] {"f1"});
VectorizedRowBatch b = schema.createRowBatch();
@@ -504,6 +505,7 @@ public class TestRowFilteringIOSkip {
.in("ridx", PredicateLeaf.Type.LONG, 0L, 1000000L, 2000000L, 3000000L)
.build();
Reader.Options options = r.options()
+ .allowSARGToFilter(false)
.useSelected(true)
.searchArgument(sarg, new String[] {"ridx"});
diff --git a/java/core/src/test/org/apache/orc/TestStringDictionary.java b/java/core/src/test/org/apache/orc/TestStringDictionary.java
index 2fc29a9c1..62ccac76a 100644
--- a/java/core/src/test/org/apache/orc/TestStringDictionary.java
+++ b/java/core/src/test/org/apache/orc/TestStringDictionary.java
@@ -621,7 +621,8 @@ public class TestStringDictionary {
SearchArgument sarg = SearchArgumentFactory.newBuilder(conf)
.lessThan("str", PredicateLeaf.Type.STRING, "row 001000")
.build();
- RecordReader recordReader = reader.rows(reader.options().searchArgument(sarg, null));
+ Reader.Options options = reader.options().searchArgument(sarg, null).allowSARGToFilter(false);
+ RecordReader recordReader = reader.rows(options);
batch = reader.getSchema().createRowBatch();
strVector = (BytesColumnVector) batch.cols[0];
long base = 0;
@@ -648,7 +649,8 @@ public class TestStringDictionary {
.lessThan("str", PredicateLeaf.Type.STRING, "row 001000")
.build();
try (Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs))) {
- try (RecordReader recordReader = reader.rows(reader.options().searchArgument(sarg, null))) {
+ Reader.Options options = reader.options().searchArgument(sarg, null).allowSARGToFilter(false);
+ try (RecordReader recordReader = reader.rows(options)) {
VectorizedRowBatch batch = reader.getSchema().createRowBatch();
BytesColumnVector strVector = (BytesColumnVector) batch.cols[0];
long base = 0;
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index 7c8d8cf1e..1c38732c1 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -2749,7 +2749,8 @@ public class TestVectorOrcFile {
rows = reader.rows(reader.options()
.range(0L, Long.MAX_VALUE)
.include(new boolean[]{true, true, true})
- .searchArgument(sarg, new String[]{null, "int1", "string1"}));
+ .searchArgument(sarg, new String[]{null, "int1", "string1"})
+ .allowSARGToFilter(false));
assertEquals(0, rows.getRowNumber());
assertTrue(rows.nextBatch(batch));
assertEquals(1000, batch.size);
@@ -4275,7 +4276,8 @@ public class TestVectorOrcFile {
rows = reader.rows(reader.options()
.range(0L, Long.MAX_VALUE)
.include(new boolean[]{true, true,true,true, true})
- .searchArgument(sarg, new String[]{null, "int1",null, "int2","string1"}));
+ .searchArgument(sarg, new String[]{null, "int1",null, "int2","string1"})
+ .allowSARGToFilter(false));
assertEquals(0, rows.getRowNumber());
assertTrue(rows.nextBatch(batch));
assertEquals(1000, batch.size);
@@ -4346,7 +4348,8 @@ public class TestVectorOrcFile {
RecordReader rows = reader.rows(reader.options()
.range(0L, Long.MAX_VALUE)
- .searchArgument(sarg, new String[]{"double1"}));
+ .searchArgument(sarg, new String[]{"double1"})
+ .allowSARGToFilter(false));
batch = reader.getSchema().createRowBatch(3500);
rows.nextBatch(batch);
@@ -4369,7 +4372,8 @@ public class TestVectorOrcFile {
rows = reader.rows(reader.options()
.range(0L, Long.MAX_VALUE)
- .searchArgument(sarg, new String[]{"float1"}));
+ .searchArgument(sarg, new String[]{"float1"})
+ .allowSARGToFilter(false));
batch = reader.getSchema().createRowBatch(3500);
rows.nextBatch(batch);
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
index d8102eddb..5349d4ead 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
@@ -412,7 +412,7 @@ public class TestOrcFileEvolution {
Reader.Options options = reader.options().schema(readTypeDescr);
if (sArg != null && sCols != null) {
- options.searchArgument(sArg, sCols);
+ options.searchArgument(sArg, sCols).allowSARGToFilter(false);
}
OrcMapredRecordReader<OrcStruct> recordReader =
diff --git a/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMapreduceOrcOutputFormat.java b/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMapreduceOrcOutputFormat.java
index e526b8630..d8c7d6fb7 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMapreduceOrcOutputFormat.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMapreduceOrcOutputFormat.java
@@ -79,6 +79,7 @@ public class TestMapreduceOrcOutputFormat {
OrcConf.MAPRED_OUTPUT_SCHEMA.setString(conf, typeStr);
conf.set("mapreduce.output.fileoutputformat.outputdir", workDir.toString());
conf.setInt(OrcConf.ROW_INDEX_STRIDE.getAttribute(), 1000);
+ conf.setBoolean(OrcConf.ALLOW_SARG_TO_FILTER.getAttribute(), false);
conf.setBoolean(OrcOutputFormat.SKIP_TEMP_DIRECTORY, true);
OutputFormat<NullWritable, OrcStruct> outputFormat =
new OrcOutputFormat<OrcStruct>();