svn commit: r722323 - in /hadoop/core/trunk: CHANGES.txt src/c++/librecordio/binarchive.cc

[cd...@apache.org]

Author: cdouglas
Date: Mon Dec  1 17:37:17 2008
New Revision: 722323

URL: http://svn.apache.org/viewvc?rev=722323&view=rev
Log:
HADOOP-4713. Fix librecordio to handle records larger than 64k. Contributed by Christian Kunz.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/c++/librecordio/binarchive.cc

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=722323&r1=722322&r2=722323&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Dec  1 17:37:17 2008
@@ -1272,6 +1272,9 @@
     HADOOP-4257. The DFS client should pick only one datanode as the candidate
     to initiate lease recovery.  (Tsz Wo (Nicholas), SZE via dhruba)
 
+    HADOOP-4713. Fix librecordio to handle records larger than 64k. (Christian
+    Kunz via cdouglas)
+
 Release 0.18.2 - 2008-11-03
 
   BUG FIXES

Modified: hadoop/core/trunk/src/c++/librecordio/binarchive.cc
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/c%2B%2B/librecordio/binarchive.cc?rev=722323&r1=722322&r2=722323&view=diff
==============================================================================
--- hadoop/core/trunk/src/c++/librecordio/binarchive.cc (original)
+++ hadoop/core/trunk/src/c++/librecordio/binarchive.cc Mon Dec  1 17:37:17 2008
@@ -161,10 +161,19 @@
   int32_t len = 0;
   ::deserializeInt(len, stream);
   if (len > 0) {
-    char buf[len];
-    stream.read((void*) buf, len);
-    std::string s(buf, len);
-    t = s;
+    // resize the string to the right length
+    t.resize(len);
+    // read into the string in 64k chunks
+    const int bufSize = 65536;
+    int offset = 0;
+    char buf[bufSize];
+    while (len > 0) {
+      int chunkLength = len > bufSize ? bufSize : len;
+      stream.read((void *)buf, chunkLength);
+      t.replace(offset, chunkLength, buf, chunkLength);
+      offset += chunkLength;
+      len -= chunkLength;
+    }
   }
 }