[incubator-hivemall] branch master updated: Increased write buffer from 1MB to 2MB

[my...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git


The following commit(s) were added to refs/heads/master by this push:
     new 1aef178  Increased write buffer from 1MB to 2MB
1aef178 is described below

commit 1aef178491297c2741f3c52418277715c8b28c6e
Author: Makoto Yui <my...@apache.org>
AuthorDate: Wed Jun 12 17:27:24 2019 +0900

    Increased write buffer from 1MB to 2MB
---
 core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
index 90ad97c..a251840 100644
--- a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
+++ b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
@@ -327,7 +327,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
             } catch (Throwable e) {
                 throw new UDFArgumentException(e);
             }
-            this.inputBuf = buf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB
+            this.inputBuf = buf = ByteBuffer.allocateDirect(2 * 1024 * 1024); // 2 MB
             this.fileIO = dst = new NioStatefulSegment(file, false);
         }
 
@@ -356,6 +356,10 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
         if (remain < requiredBytes) {
             writeBuffer(buf, dst);
         }
+        if (requiredBytes > buf.remaining()) {
+            throw new HiveException("Buffer size (2MB) for writing training example is not enough: "
+                    + NumberUtils.prettySize(requiredBytes));
+        }
 
         buf.putInt(recordBytes);
         buf.putInt(featureVector.length);