You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pdfbox.apache.org by ju...@apache.org on 2010/06/21 14:43:28 UTC
svn commit: r956548 - in /pdfbox/trunk: ./ ant/ ant/src/ ant/src/main/
ant/src/main/java/ ant/src/main/java/org/ ant/src/main/java/org/apache/
ant/src/main/java/org/apache/pdfbox/
ant/src/main/java/org/apache/pdfbox/ant/ lucene/ lucene/src/ lucene/src/...
Author: jukka
Date: Mon Jun 21 12:43:28 2010
New Revision: 956548
URL: http://svn.apache.org/viewvc?rev=956548&view=rev
Log:
PDFBOX-752: Move Lucene and Ant code into separate components
Added:
pdfbox/trunk/ant/ (with props)
pdfbox/trunk/ant/pom.xml (with props)
pdfbox/trunk/ant/src/
pdfbox/trunk/ant/src/main/
pdfbox/trunk/ant/src/main/java/
pdfbox/trunk/ant/src/main/java/org/
pdfbox/trunk/ant/src/main/java/org/apache/
pdfbox/trunk/ant/src/main/java/org/apache/pdfbox/
pdfbox/trunk/ant/src/main/java/org/apache/pdfbox/ant/
- copied from r956541, pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/ant/
pdfbox/trunk/lucene/ (with props)
pdfbox/trunk/lucene/pom.xml (with props)
pdfbox/trunk/lucene/src/
pdfbox/trunk/lucene/src/main/
pdfbox/trunk/lucene/src/main/java/
pdfbox/trunk/lucene/src/main/java/org/
pdfbox/trunk/lucene/src/main/java/org/apache/
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/
- copied from r956541, pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/searchengine/lucene/
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/
- copied from r956541, pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/searchengine/
Removed:
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/package.html
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/package.html
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/ant/
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/searchengine/
Modified:
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/IndexFiles.java
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/LucenePDFDocument.java
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/IndexFiles.java
pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/LucenePDFDocument.java
pdfbox/trunk/pdfbox/pom.xml
pdfbox/trunk/pom.xml
Propchange: pdfbox/trunk/ant/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jun 21 12:43:28 2010
@@ -0,0 +1,2 @@
+target
+.*
Added: pdfbox/trunk/ant/pom.xml
URL: http://svn.apache.org/viewvc/pdfbox/trunk/ant/pom.xml?rev=956548&view=auto
==============================================================================
--- pdfbox/trunk/ant/pom.xml (added)
+++ pdfbox/trunk/ant/pom.xml Mon Jun 21 12:43:28 2010
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one or more
+ ! contributor license agreements. See the NOTICE file distributed with
+ ! this work for additional information regarding copyright ownership.
+ ! The ASF licenses this file to You under the Apache License, Version 2.0
+ ! (the "License"); you may not use this file except in compliance with
+ ! the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.pdfbox</groupId>
+ <artifactId>pdfbox-parent</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>pdfbox-ant</artifactId>
+ <name>Apache PDFBox for Ant</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pdfbox</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ <version>1.8.1</version>
+ </dependency>
+ </dependencies>
+
+</project>
Propchange: pdfbox/trunk/ant/pom.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: pdfbox/trunk/lucene/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jun 21 12:43:28 2010
@@ -0,0 +1,2 @@
+target
+.*
Added: pdfbox/trunk/lucene/pom.xml
URL: http://svn.apache.org/viewvc/pdfbox/trunk/lucene/pom.xml?rev=956548&view=auto
==============================================================================
--- pdfbox/trunk/lucene/pom.xml (added)
+++ pdfbox/trunk/lucene/pom.xml Mon Jun 21 12:43:28 2010
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one or more
+ ! contributor license agreements. See the NOTICE file distributed with
+ ! this work for additional information regarding copyright ownership.
+ ! The ASF licenses this file to You under the Apache License, Version 2.0
+ ! (the "License"); you may not use this file except in compliance with
+ ! the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.pdfbox</groupId>
+ <artifactId>pdfbox-parent</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>pdfbox-lucene</artifactId>
+ <name>Apache PDFBox for Lucene</name>
+
+ <properties>
+ <lucene.version>2.4.1</lucene.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pdfbox</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-demos</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
+
Propchange: pdfbox/trunk/lucene/pom.xml
------------------------------------------------------------------------------
svn:eol-style = native
Modified: pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/IndexFiles.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/IndexFiles.java?rev=956548&r1=956541&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/IndexFiles.java (original)
+++ pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/IndexFiles.java Mon Jun 21 12:43:28 2010
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pdfbox.searchengine.lucene;
+package org.apache.pdfbox.lucene;
/*
* This source was originally written as an example for the lucene project.
Modified: pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/LucenePDFDocument.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/LucenePDFDocument.java?rev=956548&r1=956541&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/LucenePDFDocument.java (original)
+++ pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/lucene/LucenePDFDocument.java Mon Jun 21 12:43:28 2010
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pdfbox.searchengine.lucene;
+package org.apache.pdfbox.lucene;
import java.io.File;
import java.io.FileInputStream;
@@ -108,7 +108,7 @@ import org.apache.pdfbox.util.PDFTextStr
* @author <a href="mailto:ben@benlitchfield.com">Ben Litchfield</a>
* @version $Revision: 1.23 $
*/
-public final class LucenePDFDocument
+public class LucenePDFDocument
{
private static final char FILE_SEPARATOR = System.getProperty("file.separator").charAt(0);
Modified: pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/IndexFiles.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/IndexFiles.java?rev=956548&r1=956541&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/IndexFiles.java (original)
+++ pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/IndexFiles.java Mon Jun 21 12:43:28 2010
@@ -16,308 +16,13 @@
*/
package org.apache.pdfbox.searchengine.lucene;
-/*
- * This source was originally written as an example for the lucene project.
- * It has been modified to use PDFBox as a lucene document creator.
- * -Ben Litchfield
- *
- *====================================================================
- * The Apache Software License, Version 1.1
- *
- * Copyright (c) 2001 The Apache Software Foundation. All rights
- * reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * 3. The end-user documentation included with the redistribution,
- * if any, must include the following acknowledgment:
- * "This product includes software developed by the
- * Apache Software Foundation (http://www.apache.org/)."
- * Alternately, this acknowledgment may appear in the software itself,
- * if and wherever such third-party acknowledgments normally appear.
- *
- * 4. The names "Apache" and "Apache Software Foundation" and
- * "Apache Lucene" must not be used to endorse or promote products
- * derived from this software without prior written permission. For
- * written permission, please contact apache@apache.org.
- *
- * 5. Products derived from this software may not be called "Apache",
- * "Apache Lucene", nor may "Apache" appear in their name, without
- * prior written permission of the Apache Software Foundation.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
- * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- * ====================================================================
- *
- * This software consists of voluntary contributions made by many
- * individuals on behalf of the Apache Software Foundation. For more
- * information on the Apache Software Foundation, please see
- * <http://www.apache.org/>.
- */
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
-import org.apache.lucene.demo.HTMLDocument;
-
-import org.apache.lucene.document.Document;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
-
-import java.util.Arrays;
-
-
-import java.io.File;
-import java.io.IOException;
-
-import java.util.Date;
-
-
/**
- * This is a class that will index some files on a local filesystem. This code
- * was modified from a demo that comes with the lucene search engine.
- *
- * @author Lucene team
- * @author <a href="mailto:ben@benlitchfield.com">Ben Litchfield</a>
- *
- * @version $Revision: 1.8 $
+ * @deprecated Use {@link org.apache.pdfbox.lucene.IndexFiles} instead.
*/
-public class IndexFiles
-{
- private boolean deleting = false; // true during deletion pass
- private IndexReader reader; // existing index
- private IndexWriter writer; // new index being built
- private TermEnum uidIter; // document id iterator
-
- /**
- * This is the main entry point for the indexer.
- *
- * @param argv The command line arguments.
- */
- public static void main(String[] argv)
- {
-
- String index = "index";
- boolean create = false;
- File root = null;
-
- String usage = "org.apache.pdfbox.searchengine.lucene.IndexFiles [-create] [-index <index>] <root_directory>";
-
- if (argv.length == 0)
- {
- System.err.println("Usage: " + usage);
- return;
- }
-
- for (int i = 0; i < argv.length; i++)
- {
- if (argv[i].equals("-index"))
- { // parse -index option
- index = argv[++i];
- }
- else if (argv[i].equals("-create"))
- { // parse -create option
- create = true;
- }
- else if (i != argv.length-1)
- {
- System.err.println("Usage: " + usage);
- return;
- }
- else
- {
- System.out.println( "root=" +argv[i] );
- root = new File(argv[i]);
- }
- }
- IndexFiles indexer = new IndexFiles();
- indexer.index( root, create, index );
- }
-
- /**
- * This will index a directory.
- *
- * @param root The root directory to start indexing.
- * @param create Should we create a new index?
- * @param index The name of the index.
- */
- public void index( File root, boolean create, String index )
- {
-
- try
- {
- Date start = new Date();
+public class IndexFiles extends org.apache.pdfbox.lucene.IndexFiles {
- writer = new IndexWriter(index, new StandardAnalyzer(), create, IndexWriter.MaxFieldLength.LIMITED);
-
- if (!create)
- { // delete stale docs
- deleting = true;
- indexDocs(root, index, create);
- }
-
- indexDocs(root, index, create); // add new docs
-
- System.out.println("Optimizing index...");
- writer.optimize();
- writer.close();
-
- Date end = new Date();
-
- System.out.print(end.getTime() - start.getTime());
- System.out.println(" total milliseconds");
-
- }
- catch( Exception e )
- {
- e.printStackTrace();
- }
+ public static void main(String[] args) {
+ org.apache.pdfbox.lucene.IndexFiles.main(args);
}
- /**
- * Walk directory hierarchy in uid order, while keeping uid iterator from
- * existing index in sync. Mismatches indicate one of: (a) old documents to
- * be deleted; (b) unchanged documents, to be left alone; or (c) new
- * documents, to be indexed.
- *
- * @param file The directory to index.
- * @param index The index to add the file to.
- * @param create A flag telling if we should create the index.
- *
- * @throws Exception If there is any error indexing the directory.
- */
- private void indexDocs(File file, String index, boolean create) throws Exception
- {
- if (!create)
- { // incrementally update
-
- reader = IndexReader.open(index); // open existing index
- uidIter = reader.terms(new Term("uid", "")); // init uid iterator
-
- indexDocs(file);
-
- if (deleting)
- { // delete rest of stale docs
- while (uidIter.term() != null && uidIter.term().field().equals( "uid" ) )
- {
- System.out.println("deleting " +
- HTMLDocument.uid2url(uidIter.term().text()));
- reader.deleteDocuments(uidIter.term());
- uidIter.next();
- }
- deleting = false;
- }
-
- uidIter.close(); // close uid iterator
- reader.close(); // close existing index
-
- }
- else
- {
- indexDocs(file);
- }
- }
-
-
- private void indexDocs(File file) throws Exception
- {
- if (file.isDirectory())
- { // if a directory
- String[] files = file.list(); // list its files
- Arrays.sort(files); // sort the files
- for (int i = 0; i < files.length; i++) // recursively index them
- {
- indexDocs(new File(file, files[i]));
- }
- }
- else
- {
- if (uidIter != null)
- {
- String uid = HTMLDocument.uid(file); // construct uid for doc
-
- while( uidIter.term() != null &&
- uidIter.term().field().equals( "uid" ) &&
- uidIter.term().text().compareTo(uid) < 0)
- {
- if (deleting)
- { // delete stale docs
- System.out.println("deleting " +
- HTMLDocument.uid2url(uidIter.term().text()));
- reader.deleteDocuments(uidIter.term());
- }
- uidIter.next();
- }
- if( uidIter.term() != null &&
- uidIter.term().field().equals( "uid" ) &&
- uidIter.term().text().compareTo(uid) == 0)
- {
- System.out.println( "Next uid=" +uidIter );
- uidIter.next(); // keep matching docs
- }
- }
- else
- {
- try
- {
- addDocument( file );
- }
- catch( IOException e )
- {
- //catch exception and move onto the next document
- System.out.println( e.getMessage() );
- }
- }
- }
- }
-
- private void addDocument( File file ) throws IOException, InterruptedException
- {
- String path = file.getName().toUpperCase();
- Document doc = null;
- //Gee, this would be a great place for a command pattern
- if( path.endsWith(".HTML") || // index .html files
- path.endsWith(".HTM") || // index .htm files
- path.endsWith(".TXT"))
- {
- System.out.println( "Indexing Text document: " + file );
- doc = HTMLDocument.Document(file);
- }
- else if( path.endsWith( ".PDF" ) )
- {
- System.out.println( "Indexing PDF document: " + file );
- doc = LucenePDFDocument.getDocument( file );
- }
- else
- {
- System.out.println( "Skipping " + file );
- }
-
- if( doc != null )
- {
- writer.addDocument(doc);
- }
- }
}
Modified: pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/LucenePDFDocument.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/LucenePDFDocument.java?rev=956548&r1=956541&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/LucenePDFDocument.java (original)
+++ pdfbox/trunk/lucene/src/main/java/org/apache/pdfbox/searchengine/lucene/LucenePDFDocument.java Mon Jun 21 12:43:28 2010
@@ -16,484 +16,9 @@
*/
package org.apache.pdfbox.searchengine.lucene;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.util.Calendar;
-
-import java.net.URL;
-import java.net.URLConnection;
-
-import java.util.Date;
-
-import org.apache.lucene.document.DateTools;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDDocumentInformation;
-
-import org.apache.pdfbox.exceptions.CryptographyException;
-import org.apache.pdfbox.exceptions.InvalidPasswordException;
-
-import org.apache.pdfbox.util.PDFTextStripper;
-
/**
- * This class is used to create a document for the lucene search engine.
- * This should easily plug into the IndexHTML or IndexFiles that comes with
- * the lucene project. This class will populate the following fields.
- * <table>
- * <tr>
- * <th>Lucene Field Name</th>
- * <th>Description</th>
- * </tr>
- * <tr>
- * <td>path</td>
- * <td>File system path if loaded from a file</td>
- * </tr>
- * <tr>
- * <td>url</td>
- * <td>URL to PDF document</td>
- * </tr>
- * <tr>
- * <td>contents</td>
- * <td>Entire contents of PDF document, indexed but not stored</td>
- * </tr>
- * <tr>
- * <td>summary</td>
- * <td>First 500 characters of content</td>
- * </tr>
- * <tr>
- * <td>modified</td>
- * <td>The modified date/time according to the url or path</td>
- * </tr>
- * <tr>
- * <td>uid</td>
- * <td>A unique identifier for the Lucene document.</td>
- * </tr>
- * <tr>
- * <td>CreationDate</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>Creator</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>Keywords</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>ModificationDate</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>Producer</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>Subject</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * <tr>
- * <td>Trapped</td>
- * <td>From PDF meta-data if available</td>
- * </tr>
- * </table>
- *
- * @author <a href="mailto:ben@benlitchfield.com">Ben Litchfield</a>
- * @version $Revision: 1.23 $
+ * @deprecated Use {@link org.apache.pdfbox.lucene.LucenePDFDocument} instead.
*/
-public final class LucenePDFDocument
-{
- private static final char FILE_SEPARATOR = System.getProperty("file.separator").charAt(0);
-
- // given caveat of increased search times when using
- //MICROSECOND, only use SECOND by default
- private DateTools.Resolution dateTimeResolution = DateTools.Resolution.SECOND;
-
- private PDFTextStripper stripper = null;
-
- /**
- * Constructor.
- */
- public LucenePDFDocument()
- {
- }
-
- /**
- * Set the text stripper that will be used during extraction.
- *
- * @param aStripper The new pdf text stripper.
- */
- public void setTextStripper( PDFTextStripper aStripper )
- {
- stripper = aStripper;
- }
-
- /**
- * Get the Lucene data time resolution.
- *
- * @return current date/time resolution
- */
- public DateTools.Resolution getDateTimeResolution()
- {
- return dateTimeResolution;
- }
-
- /**
- * Set the Lucene data time resolution.
- *
- * @param resolution set new date/time resolution
- */
- public void setDateTimeResolution( DateTools.Resolution resolution )
- {
- dateTimeResolution = resolution;
- }
-
- //
- // compatibility methods for lucene-1.9+
- //
- private String timeToString( long time )
- {
- return DateTools.timeToString( time, dateTimeResolution );
- }
-
- private void addKeywordField( Document document, String name, String value )
- {
- if ( value != null )
- {
- document.add( new Field( name, value, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
- }
- }
-
- private void addTextField( Document document, String name, Reader value )
- {
- if ( value != null )
- {
- document.add( new Field( name, value ) );
- }
- }
-
- private void addTextField( Document document, String name, String value )
- {
- if ( value != null )
- {
- document.add( new Field( name, value, Field.Store.YES, Field.Index.ANALYZED ) );
- }
- }
-
- private void addTextField( Document document, String name, Date value )
- {
- if ( value != null )
- {
- addTextField( document, name, DateTools.dateToString( value, dateTimeResolution ) );
- }
- }
-
- private void addTextField( Document document, String name, Calendar value )
- {
- if ( value != null )
- {
- addTextField( document, name, value.getTime() );
- }
- }
-
- private static void addUnindexedField( Document document, String name, String value )
- {
- if ( value != null )
- {
- document.add( new Field( name, value, Field.Store.YES, Field.Index.NO ) );
- }
- }
-
- private void addUnstoredKeywordField( Document document, String name, String value )
- {
- if ( value != null )
- {
- document.add( new Field( name, value, Field.Store.NO, Field.Index.NOT_ANALYZED ) );
- }
- }
-
- /**
- * Convert the PDF stream to a lucene document.
- *
- * @param is The input stream.
- * @return The input stream converted to a lucene document.
- * @throws IOException If there is an error converting the PDF.
- */
- public Document convertDocument( InputStream is ) throws IOException
- {
- Document document = new Document();
- addContent( document, is, "<inputstream>" );
- return document;
-
- }
-
- /**
- * This will take a reference to a PDF document and create a lucene document.
- *
- * @param file A reference to a PDF document.
- * @return The converted lucene document.
- *
- * @throws IOException If there is an exception while converting the document.
- */
- public Document convertDocument( File file ) throws IOException
- {
- Document document = new Document();
-
- // Add the url as a field named "url". Use an UnIndexed field, so
- // that the url is just stored with the document, but is not searchable.
- addUnindexedField( document, "path", file.getPath() );
- addUnindexedField( document, "url", file.getPath().replace(FILE_SEPARATOR, '/') );
-
- // Add the last modified date of the file a field named "modified". Use a
- // Keyword field, so that it's searchable, but so that no attempt is made
- // to tokenize the field into words.
- addKeywordField( document, "modified", timeToString( file.lastModified() ) );
-
- String uid = file.getPath().replace(FILE_SEPARATOR,'\u0000')
- + "\u0000"
- + timeToString( file.lastModified() );
-
- // Add the uid as a field, so that index can be incrementally maintained.
- // This field is not stored with document, it is indexed, but it is not
- // tokenized prior to indexing.
- addUnstoredKeywordField( document, "uid", uid );
-
- FileInputStream input = null;
- try
- {
- input = new FileInputStream( file );
- addContent( document, input, file.getPath() );
- }
- finally
- {
- if( input != null )
- {
- input.close();
- }
- }
-
-
- // return the document
-
- return document;
- }
-
- /**
- * Convert the document from a PDF to a lucene document.
- *
- * @param url A url to a PDF document.
- * @return The PDF converted to a lucene document.
- * @throws IOException If there is an error while converting the document.
- */
- public Document convertDocument( URL url ) throws IOException
- {
- Document document = new Document();
- URLConnection connection = url.openConnection();
- connection.connect();
- // Add the url as a field named "url". Use an UnIndexed field, so
- // that the url is just stored with the document, but is not searchable.
- addUnindexedField( document, "url", url.toExternalForm() );
-
- // Add the last modified date of the file a field named "modified". Use a
- // Keyword field, so that it's searchable, but so that no attempt is made
- // to tokenize the field into words.
- addKeywordField( document, "modified", timeToString(connection.getLastModified() ) );
-
- String uid = url.toExternalForm().replace(FILE_SEPARATOR, '\u0000')
- + "\u0000"
- + timeToString( connection.getLastModified() );
-
- // Add the uid as a field, so that index can be incrementally maintained.
- // This field is not stored with document, it is indexed, but it is not
- // tokenized prior to indexing.
- addUnstoredKeywordField( document, "uid", uid );
-
- InputStream input = null;
- try
- {
- input = connection.getInputStream();
- addContent( document, input,url.toExternalForm() );
- }
- finally
- {
- if( input != null )
- {
- input.close();
- }
- }
-
- // return the document
- return document;
- }
-
- /**
- * This will get a lucene document from a PDF file.
- *
- * @param is The stream to read the PDF from.
- *
- * @return The lucene document.
- *
- * @throws IOException If there is an error parsing or indexing the document.
- */
- public static Document getDocument( InputStream is ) throws IOException
- {
- LucenePDFDocument converter = new LucenePDFDocument();
- return converter.convertDocument( is );
- }
-
- /**
- * This will get a lucene document from a PDF file.
- *
- * @param file The file to get the document for.
- *
- * @return The lucene document.
- *
- * @throws IOException If there is an error parsing or indexing the document.
- */
- public static Document getDocument( File file ) throws IOException
- {
- LucenePDFDocument converter = new LucenePDFDocument();
- return converter.convertDocument( file );
- }
-
- /**
- * This will get a lucene document from a PDF file.
- *
- * @param url The file to get the document for.
- *
- * @return The lucene document.
- *
- * @throws IOException If there is an error parsing or indexing the document.
- */
- public static Document getDocument( URL url ) throws IOException
- {
- LucenePDFDocument converter = new LucenePDFDocument();
- return converter.convertDocument( url );
- }
-
- /**
- * This will add the contents to the lucene document.
- *
- * @param document The document to add the contents to.
- * @param is The stream to get the contents from.
- * @param documentLocation The location of the document, used just for debug messages.
- *
- * @throws IOException If there is an error parsing the document.
- */
- private void addContent( Document document, InputStream is, String documentLocation ) throws IOException
- {
- PDDocument pdfDocument = null;
- try
- {
- pdfDocument = PDDocument.load( is );
-
- if( pdfDocument.isEncrypted() )
- {
- //Just try using the default password and move on
- pdfDocument.decrypt( "" );
- }
-
- //create a writer where to append the text content.
- StringWriter writer = new StringWriter();
- if( stripper == null )
- {
- stripper = new PDFTextStripper();
- }
- else
- {
- stripper.resetEngine();
- }
- stripper.writeText( pdfDocument, writer );
-
- // Note: the buffer to string operation is costless;
- // the char array value of the writer buffer and the content string
- // is shared as long as the buffer content is not modified, which will
- // not occur here.
- String contents = writer.getBuffer().toString();
-
- StringReader reader = new StringReader( contents );
-
- // Add the tag-stripped contents as a Reader-valued Text field so it will
- // get tokenized and indexed.
- addTextField( document, "contents", reader );
-
- PDDocumentInformation info = pdfDocument.getDocumentInformation();
- if( info != null )
- {
- addTextField( document, "Author", info.getAuthor() );
- try
- {
- addTextField( document, "CreationDate", info.getCreationDate() );
- }
- catch( IOException io )
- {
- //ignore, bad date but continue with indexing
- }
- addTextField( document, "Creator", info.getCreator() );
- addTextField( document, "Keywords", info.getKeywords() );
- try
- {
- addTextField( document, "ModificationDate", info.getModificationDate() );
- }
- catch( IOException io )
- {
- //ignore, bad date but continue with indexing
- }
- addTextField( document, "Producer", info.getProducer() );
- addTextField( document, "Subject", info.getSubject() );
- addTextField( document, "Title", info.getTitle() );
- addTextField( document, "Trapped", info.getTrapped() );
- }
- int summarySize = Math.min( contents.length(), 500 );
- String summary = contents.substring( 0, summarySize );
- // Add the summary as an UnIndexed field, so that it is stored and returned
- // with hit documents for display.
- addUnindexedField( document, "summary", summary );
- }
- catch( CryptographyException e )
- {
- throw new IOException( "Error decrypting document(" + documentLocation + "): " + e );
- }
- catch( InvalidPasswordException e )
- {
- //they didn't suppply a password and the default of "" was wrong.
- throw new IOException(
- "Error: The document(" + documentLocation +
- ") is encrypted and will not be indexed." );
- }
- finally
- {
- if( pdfDocument != null )
- {
- pdfDocument.close();
- }
- }
- }
-
- /**
- * This will test creating a document.
- *
- * usage: java pdfparser.searchengine.lucene.LucenePDFDocument <pdf-document>
- *
- * @param args command line arguments.
- *
- * @throws IOException If there is an error.
- */
- public static void main( String[] args ) throws IOException
- {
- if( args.length != 1 )
- {
- String us = LucenePDFDocument.class.getName();
- System.err.println( "usage: java " + us + " <pdf-document>" );
- System.exit( 1 );
- }
- System.out.println( "Document=" + getDocument( new File( args[0] ) ) );
- }
+public class LucenePDFDocument
+ extends org.apache.pdfbox.lucene.LucenePDFDocument {
}
Modified: pdfbox/trunk/pdfbox/pom.xml
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/pom.xml?rev=956548&r1=956547&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/pom.xml (original)
+++ pdfbox/trunk/pdfbox/pom.xml Mon Jun 21 12:43:28 2010
@@ -53,24 +53,6 @@
<version>1.1.1</version>
</dependency>
<dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>2.4.1</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-demos</artifactId>
- <version>2.4.1</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.ant</groupId>
- <artifactId>ant</artifactId>
- <version>1.7.1</version>
- <optional>true</optional>
- </dependency>
- <dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcmail-jdk15</artifactId>
<version>1.44</version>
Modified: pdfbox/trunk/pom.xml
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pom.xml?rev=956548&r1=956547&r2=956548&view=diff
==============================================================================
--- pdfbox/trunk/pom.xml (original)
+++ pdfbox/trunk/pom.xml Mon Jun 21 12:43:28 2010
@@ -47,6 +47,8 @@
<module>fontbox</module>
<module>jempbox</module>
<module>pdfbox</module>
+ <module>lucene</module>
+ <module>ant</module>
<module>app</module>
</modules>