[orc] branch main updated: ORC-913: Support data/format/compress options in Spark benchmark (#825)

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/main by this push:
     new 4f089c2  ORC-913: Support data/format/compress options in Spark benchmark (#825)
4f089c2 is described below

commit 4f089c29a4d7fa09f711d5979628f3ab6459117c
Author: William Hyun <wi...@apache.org>
AuthorDate: Sat Aug 7 09:03:40 2021 -0700

    ORC-913: Support data/format/compress options in Spark benchmark (#825)
    
    ### What changes were proposed in this pull request?
    This PR aims to support data/format/compress options in Spark benchmark.
    
    ### Why are the changes needed?
    This will make Spark benchmark more controllable.
    
    This extension does not appear in -h option, however, it will work.
    
    ### How was this patch tested?
    Manual
    
    ```
    java -jar spark/target/orc-benchmarks-spark-1.8.0-SNAPSHOT.jar spark data -d sales -c snappy -f orc
    ```
---
 .../org/apache/orc/bench/core/convert/GenerateVariants.java  |  2 +-
 .../src/java/org/apache/orc/bench/spark/SparkBenchmark.java  | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java b/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
index 1c95434..eb51627 100644
--- a/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
+++ b/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
@@ -223,7 +223,7 @@ public class GenerateVariants implements OrcBenchmark {
     }
   }
 
-  static CommandLine parseCommandLine(String[] args) throws ParseException {
+  public static CommandLine parseCommandLine(String[] args) throws ParseException {
     Options options = new Options()
         .addOption("h", "help", false, "Provide help")
         .addOption("c", "compress", true, "List of compression")
diff --git a/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java b/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
index f01c140..f9c817d 100644
--- a/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
+++ b/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
@@ -19,6 +19,7 @@
 package org.apache.orc.bench.spark;
 
 import com.google.auto.service.AutoService;
+import org.apache.commons.cli.CommandLine;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -27,6 +28,7 @@ import org.apache.orc.TypeDescription;
 import org.apache.orc.bench.core.OrcBenchmark;
 import org.apache.orc.bench.core.IOCounters;
 import org.apache.orc.bench.core.Utilities;
+import org.apache.orc.bench.core.convert.GenerateVariants;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.avro.AvroFileFormat;
 import org.apache.spark.sql.catalyst.InternalRow;
@@ -52,6 +54,7 @@ import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.infra.Blackhole;
 import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 import scala.Function1;
 
 import java.io.IOException;
@@ -87,7 +90,14 @@ public class SparkBenchmark implements OrcBenchmark {
 
   @Override
   public void run(String[] args) throws Exception {
-    new Runner(Utilities.parseOptions(args, this.getClass())).run();
+    CommandLine cmds = GenerateVariants.parseCommandLine(args);
+    new Runner(new OptionsBuilder()
+        .parent(Utilities.parseOptions(args, this.getClass()))
+        .param("compression", cmds.getOptionValue("compress", "none,gz,snappy").split(","))
+        .param("dataset", cmds.getOptionValue("data", "taxi,sales,github").split(","))
+        .param("format", cmds.getOptionValue("format", "orc,parquet,json").split(","))
+        .build()
+    ).run();
   }
 
   @State(Scope.Thread)