[orc] branch master updated: ORC-476: Make SearchAgument kryo buffer size configurable

[om...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

omalley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/master by this push:
     new 0b731f4  ORC-476: Make SearchAgument kryo buffer size configurable
0b731f4 is described below

commit 0b731f421fc95891e6655457be30e50f3cde4aee
Author: Dhruve Ashar <da...@yahoo-inc.com>
AuthorDate: Wed Mar 6 17:48:11 2019 -0600

    ORC-476: Make SearchAgument kryo buffer size configurable
    
    Fixes #372
    
    Signed-off-by: Owen O'Malley <om...@apache.org>
---
 java/core/src/java/org/apache/orc/OrcConf.java                    | 2 ++
 java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
index 23e301e..a6fbad1 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -138,6 +138,8 @@ public enum OrcConf {
           "being the first column, 1 being the next, and so on. ."),
   KRYO_SARG("orc.kryo.sarg", "orc.kryo.sarg", null,
       "The kryo and base64 encoded SearchArgument for predicate pushdown."),
+  KRYO_SARG_BUFFER("orc.kryo.sarg.buffer", null, 8192,
+      "The kryo buffer size for SearchArgument for predicate pushdown."),
   SARG_COLUMNS("orc.sarg.column.names", "org.sarg.column.names", null,
       "The list of column names for the SearchArgument."),
   FORCE_POSITIONAL_EVOLUTION("orc.force.positional.evolution",
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
index c6975d8..97d6164 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
@@ -51,6 +51,8 @@ import org.apache.orc.TypeDescription;
 public class OrcInputFormat<V extends WritableComparable>
     extends FileInputFormat<NullWritable, V> {
 
+  private final static int KRYO_SARG_MAX_BUFFER = 16777216;
+
   /**
    * Convert a string with a comma separated list of column ids into the
    * array of boolean that match the schemas.
@@ -90,7 +92,8 @@ public class OrcInputFormat<V extends WritableComparable>
   public static void setSearchArgument(Configuration conf,
                                        SearchArgument sarg,
                                        String[] columnNames) {
-    Output out = new Output(100000);
+    int bufferSize = (int)OrcConf.KRYO_SARG_BUFFER.getLong(conf);
+    Output out = new Output(bufferSize, KRYO_SARG_MAX_BUFFER);
     new Kryo().writeObject(out, sarg);
     OrcConf.KRYO_SARG.setString(conf, Base64.encodeBase64String(out.toBytes()));
     StringBuilder buffer = new StringBuilder();