You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2015/03/07 05:50:14 UTC
svn commit: r1664792 - in /nutch/trunk: CHANGES.txt
src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
Author: mattmann
Date: Sat Mar 7 04:50:14 2015
New Revision: 1664792
URL: http://svn.apache.org/r1664792
Log:
Fix for NUTCH-1954: FilenameTooLong error appears in CommonCrawlDumper.
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1664792&r1=1664791&r2=1664792&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sat Mar 7 04:50:14 2015
@@ -2,6 +2,8 @@ Nutch Change Log
Nutch Current Development 1.10-SNAPSHOT
+* NUTCH-1954 FilenameTooLong error appears in CommonCrawlDumper (mattmann)
+
* NUTCH-1949 Dump out the Nutch data into the Common Crawl format (Giuseppe Totaro via lewismc)
* NUTCH-1950 File name too long (Jiaheng Zhang, Chong Li via mattmann)
Modified: nutch/trunk/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java?rev=1664792&r1=1664791&r2=1664792&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java Sat Mar 7 04:50:14 2015
@@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import java.security.MessageDigest;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -369,7 +370,22 @@ public class CommonCrawlDataDumper {
}
else {
LOG.info("Writing: [" + outputFullPath + "]");
- IOUtils.copy(new ByteArrayInputStream(byteData), new FileOutputStream(outputFile));
+ try{
+ IOUtils.copy(new ByteArrayInputStream(byteData), new FileOutputStream(outputFile));
+ }
+ catch (Exception e){
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(outputFullPath.getBytes());
+ byte[] digest = md.digest();
+ StringBuffer sb = new StringBuffer();
+ for (byte b : digest) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ outputFullPath = outputFullPath.substring(0, 32) + "_" + sb.toString();
+ File newOutPutFile = new File(outputFullPath);
+ IOUtils.copy(new ByteArrayInputStream(byteData), new FileOutputStream(newOutPutFile));
+ LOG.info("File name is too long. Truncated and MD5 appended.");
+ }
}
}
else {