You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ab...@apache.org on 2009/02/06 14:09:09 UTC
svn commit: r741558 - in /lucene/nutch/trunk: ./ src/plugin/parse-pdf/
src/plugin/parse-pdf/lib/ src/plugin/parse-pdf/sample/
src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/
src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/
Author: ab
Date: Fri Feb 6 13:09:07 2009
New Revision: 741558
URL: http://svn.apache.org/viewvc?rev=741558&view=rev
Log:
NUTCH-643 ClassCastException in PDF parser, upgrade to unofficial PDFBox 0.7.4
Added:
lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf (with props)
Removed:
lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.1.0-dev.jar
lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.3.jar
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/plugin/parse-pdf/build.xml
lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml
lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Fri Feb 6 13:09:07 2009
@@ -333,6 +333,8 @@
124. NUTCH-671 - JSP errors in Nutch searcher webapp (Edwin Chu via ab)
+125. NUTCH-643 - ClassCastException in PDF parser (Guillaume Smet, ab)
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/build.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/build.xml?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/build.xml (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/build.xml Fri Feb 6 13:09:07 2009
@@ -28,6 +28,10 @@
<!-- for junit test -->
<mkdir dir="${build.test}/data"/>
- <copy file="sample/pdftest.pdf" todir="${build.test}/data"/>
+ <copy todir="${build.test}/data">
+ <fileset dir="sample">
+ <include name="*.pdf"/>
+ </fileset>
+ </copy>
</project>
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt Fri Feb 6 13:09:07 2009
@@ -0,0 +1,25 @@
+Copyright (c) 2006-2007, www.jempbox.org
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+3. Neither the name of pdfbox; nor the names of its
+ contributors may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt Fri Feb 6 13:09:07 2009
@@ -0,0 +1,6 @@
+This plugin uses an unreleased version of PDFBox, obtained from SourceForce
+CVS repository (HEAD revision as of Feb 3 2009), because the last released
+version contains a bug related to document decryption (see NUTCH-643).
+
+The libraries should be updated to a regular PDFBox release whenever that
+becomes available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt Fri Feb 6 13:09:07 2009
@@ -0,0 +1,7 @@
+Copyright (c) 2000 - 2008 The Legion Of The Bouncy Castle (http://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml Fri Feb 6 13:09:07 2009
@@ -26,8 +26,12 @@
<library name="parse-pdf.jar">
<export name="*"/>
</library>
- <library name="PDFBox-0.7.3.jar"/>
- <library name="FontBox-0.1.0-dev.jar"/>
+ <library name="PDFBox-0.7.4-dev.jar"/>
+ <library name="FontBox-0.2.0-dev.jar"/>
+ <library name="JempBox-0.2.0-dev.jar"/>
+ <library name="bcprov-jdk14-132.jar"/>
+ <library name="jai_codec.jar"/>
+ <library name="jai_core.jar"/>
</runtime>
<requires>
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt Fri Feb 6 13:09:07 2009
@@ -1,10 +1,5 @@
-20040710, John Xing
-
-./pdftest.pdf: a small pdf file picked up from internet.
-
-Used for junit test by ./src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
-
-It contains text:
+1. pdftest.pdf:
+A small pdf file picked up from internet. It contains text:
E:\M55\!\1725.fm 2003-01-01 18:15 P Tagg, IPM, University of Liverpool
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
@@ -12,3 +7,7 @@
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
+
+2. encrypt.pdf:
+The same file as above, encrypted with owner's password 'owner'
+and empty user's password.
Added: lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf
------------------------------------------------------------------------------
svn:mime-type = application/pdf
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java Fri Feb 6 13:09:07 2009
@@ -17,14 +17,14 @@
package org.apache.nutch.parse.pdf;
-import org.pdfbox.encryption.DocumentEncryption;
import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation;
+import org.pdfbox.pdmodel.encryption.BadSecurityHandlerException;
+import org.pdfbox.pdmodel.encryption.StandardDecryptionMaterial;
import org.pdfbox.util.PDFTextStripper;
import org.pdfbox.exceptions.CryptographyException;
-import org.pdfbox.exceptions.InvalidPasswordException;
// Commons Logging imports
import org.apache.commons.logging.Log;
@@ -93,9 +93,8 @@
pdf = parser.getPDDocument();
if (pdf.isEncrypted()) {
- DocumentEncryption decryptor = new DocumentEncryption(pdf);
//Just try using the default password and move on
- decryptor.decryptDocument("");
+ pdf.openProtection(new StandardDecryptionMaterial(""));
}
// collect text
@@ -106,7 +105,7 @@
PDDocumentInformation info = pdf.getDocumentInformation();
title = info.getTitle();
// more useful info, currently not used. please keep them for future use.
- metadata.add(Metadata.PAGE_COUNT, String.valueOf(pdf.getPageCount()));
+ metadata.add(Metadata.PAGE_COUNT, String.valueOf(pdf.getNumberOfPages()));
metadata.add(Metadata.AUTHOR, info.getAuthor());
metadata.add(Metadata.SUBJECT, info.getSubject());
metadata.add(Metadata.KEYWORDS, info.getKeywords());
@@ -122,9 +121,9 @@
} catch (CryptographyException e) {
return new ParseStatus(ParseStatus.FAILED,
"Error decrypting document. " + e).getEmptyParseResult(content.getUrl(), getConf());
- } catch (InvalidPasswordException e) {
+ } catch (BadSecurityHandlerException e) {
return new ParseStatus(ParseStatus.FAILED,
- "Can't decrypt document - invalid password. " + e).getEmptyParseResult(content.getUrl(), getConf());
+ "Error decrypting document. " + e).getEmptyParseResult(content.getUrl(), getConf());
} catch (Exception e) { // run time exception
if (LOG.isWarnEnabled()) {
LOG.warn("General exception in PDF parser: "+e.getMessage());
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java Fri Feb 6 13:09:07 2009
@@ -46,7 +46,10 @@
// Make sure sample files are copied to "test.data" as specified in
// ./src/plugin/parse-pdf/build.xml during plugin compilation.
// Check ./src/plugin/parse-pdf/sample/README.txt for what they are.
- private String[] sampleFiles = {"pdftest.pdf"};
+ private String[] sampleFiles = {
+ "pdftest.pdf",
+ "encrypted.pdf"
+ };
private String expectedText = "A VERY SMALL PDF FILE";
@@ -64,7 +67,7 @@
Content content;
Parse parse;
- for (int i=0; i<sampleFiles.length; i++) {
+ for (int i = 0; i < sampleFiles.length; i++) {
urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
Configuration conf = NutchConfiguration.create();