[GitHub] jon-wei closed pull request #6080: Accept total rows over Integer.MAX_VALUE.

[GitBox <gi...@apache.org>]

jon-wei closed pull request #6080: Accept total rows over Integer.MAX_VALUE.
URL: https://github.com/apache/incubator-druid/pull/6080
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
index 55c9a3d4132..38caea85818 100644
--- a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
+++ b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
@@ -577,7 +577,7 @@ public DimValueCount apply(DimValueCount arg1, DimValueCount arg2)
               }
 
               // Respect "poisoning" (negative values mean we can't use this dimension)
-              final int newNumRows = (arg1.numRows >= 0 && arg2.numRows >= 0 ? arg1.numRows + arg2.numRows : -1);
+              final long newNumRows = (arg1.numRows >= 0 && arg2.numRows >= 0 ? arg1.numRows + arg2.numRows : -1);
               return new DimValueCount(arg1.dim, arg1.value, newNumRows);
             }
           }
@@ -620,7 +620,7 @@ protected void innerReduce(
 
       // First DVC should be the total row count indicator
       final DimValueCount firstDvc = iterator.next();
-      final int totalRows = firstDvc.numRows;
+      final long totalRows = firstDvc.numRows;
 
       if (!firstDvc.dim.equals("") || !firstDvc.value.equals("")) {
         throw new IllegalStateException("WTF?! Expected total row indicator on first k/v pair!");
@@ -899,9 +899,9 @@ public long getDistanceSquaredFromTarget(long target)
       return distance;
     }
 
-    public int getRows()
+    public long getRows()
     {
-      int sum = 0;
+      long sum = 0;
       for (final DimPartition dimPartition : partitions) {
         sum += dimPartition.rows;
       }
@@ -913,16 +913,16 @@ public int getRows()
   {
     public ShardSpec shardSpec = null;
     public int cardinality = 0;
-    public int rows = 0;
+    public long rows = 0;
   }
 
   private static class DimValueCount
   {
     public final String dim;
     public final String value;
-    public final int numRows;
+    public final long numRows;
 
-    private DimValueCount(String dim, String value, int numRows)
+    private DimValueCount(String dim, String value, long numRows)
     {
       this.dim = dim;
       this.value = value;
@@ -938,7 +938,7 @@ public static DimValueCount fromText(Text text)
     {
       final Iterator<String> splits = TAB_SPLITTER.limit(3).split(text.toString()).iterator();
       final String dim = splits.next();
-      final int numRows = Integer.parseInt(splits.next());
+      final long numRows = Long.parseLong(splits.next());
       final String value = splits.next();
 
       return new DimValueCount(dim, value, numRows);
diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
index 739bd9effca..b3dde787ea7 100644
--- a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
+++ b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
@@ -52,12 +52,12 @@
   private DatasourceIngestionSpec spec;
   private IngestSegmentFirehose firehose;
 
-  private int rowNum;
+  private long rowNum;
   private Row currRow;
 
   private List<QueryableIndex> indexes = Lists.newArrayList();
   private List<File> tmpSegmentDirs = Lists.newArrayList();
-  private int numRows;
+  private long numRows;
 
   @Override
   public void initialize(InputSplit split, final TaskAttemptContext context) throws IOException


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org