[orc] branch branch-1.7 updated: ORC-849: Core Benchmark Cleanup (#752)

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/branch-1.7 by this push:
     new 54a0b7a  ORC-849: Core Benchmark Cleanup (#752)
54a0b7a is described below

commit 54a0b7a8ac3aa7c4b3002bdfcc39f366306920a0
Author: belugabehr <12...@users.noreply.github.com>
AuthorDate: Thu Jul 15 23:18:15 2021 -0400

    ORC-849: Core Benchmark Cleanup (#752)
    
    ### What changes were proposed in this pull request?
    Add additional output. Code cleanup.
    
    ### Why are the changes needed?
    Adding this logging helped me identify an error condition.
    
    ### How was this patch tested?
    This is benchmark code. No change in functionality. Use existing unit tests.
    
    (cherry picked from commit 2349a599133bef167519479e100349db82cbc00e)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../orc/bench/core/convert/GenerateVariants.java   | 24 ++++++++++++++--------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java b/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
index d203bfd..eb51627 100644
--- a/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
+++ b/java/bench/core/src/java/org/apache/orc/bench/core/convert/GenerateVariants.java
@@ -46,6 +46,7 @@ import org.apache.orc.bench.core.convert.parquet.ParquetReader;
 import org.apache.orc.bench.core.convert.parquet.ParquetWriter;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * A tool to create the different variants that we need to benchmark against.
@@ -120,10 +121,12 @@ public class GenerateVariants implements OrcBenchmark {
     long records = Long.parseLong(cli.getOptionValue("sales", "25000000"));
     Configuration conf = new Configuration();
     Path root = new Path(cli.getArgs()[0]);
-    for(String data: dataList) {
+
+    for (final String data: dataList) {
+      System.out.println("Processing " + data + " " + Arrays.toString(formatList));
+
       // Set up the reader
       TypeDescription schema = Utilities.loadSchema(data + ".schema");
-      BatchReader reader = createReader(root, data, schema, conf, records);
 
       // Set up the writers for each combination
       BatchWriter[] writers = new BatchWriter[compressList.length * formatList.length];
@@ -139,15 +142,18 @@ public class GenerateVariants implements OrcBenchmark {
         }
       }
 
-      // Copy the rows
-      VectorizedRowBatch batch = schema.createRowBatch();
-      while (reader.nextBatch(batch)) {
-        for(BatchWriter writer: writers) {
-          writer.writeBatch(batch);
+      // Copy the rows from Reader
+      try (BatchReader reader = createReader(root, data, schema, conf, records)) {
+        VectorizedRowBatch batch = schema.createRowBatch();
+        while (reader.nextBatch(batch)) {
+          for (BatchWriter writer : writers) {
+            writer.writeBatch(batch);
+          }
         }
       }
-      reader.close();
-      for(BatchWriter writer: writers) {
+
+      // Close all the writers
+      for (BatchWriter writer : writers) {
         writer.close();
       }
     }