You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lenya.apache.org by mi...@apache.org on 2004/08/02 02:19:42 UTC
cvs commit: cocoon-lenya/src/java/org/apache/lenya/lucene/index AbstractIndexer.java AbstractDocumentCreator.java
michi 2004/08/01 17:19:42
Modified: src/java/org/apache/lenya/cms/task LuceneTask.java
src/java/org/apache/lenya/lucene/index AbstractIndexer.java
AbstractDocumentCreator.java
Log:
incrementally indexing finished
Revision Changes Path
1.2 +4 -4 cocoon-lenya/src/java/org/apache/lenya/cms/task/LuceneTask.java
Index: LuceneTask.java
===================================================================
RCS file: /home/cvs/cocoon-lenya/src/java/org/apache/lenya/cms/task/LuceneTask.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- LuceneTask.java 29 Jul 2004 05:49:24 -0000 1.1
+++ LuceneTask.java 2 Aug 2004 00:19:42 -0000 1.2
@@ -22,16 +22,16 @@
log.debug("Servlet context path: " + servletContextPath);
String publicationId = getParameters().getParameter("publication-id");
- log.error("Publication ID: " + publicationId);
+ log.debug("Publication ID: " + publicationId);
String files = getParameters().getParameter("properties.files2index");
- log.error("Files 2 index: " + files);
+ log.debug("Files 2 index: " + files);
String luceneConfig = getParameters().getParameter("config");
log.debug("Lucene configuration: " + luceneConfig);
String luceneConfigAbsolutePath = servletContextPath + File.separator + "lenya" + File.separator + "pubs" + File.separator + publicationId + File.separator + luceneConfig;
- log.error("Lucene configuration: " + luceneConfigAbsolutePath);
+ log.debug("Lucene configuration: " + luceneConfigAbsolutePath);
Indexer indexer = Index.getIndexer(luceneConfigAbsolutePath);
indexer.indexDocument(new File(files));
1.21 +333 -331 cocoon-lenya/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java
Index: AbstractIndexer.java
===================================================================
RCS file: /home/cvs/cocoon-lenya/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -r1.20 -r1.21
--- AbstractIndexer.java 1 Aug 2004 10:48:06 -0000 1.20
+++ AbstractIndexer.java 2 Aug 2004 00:19:42 -0000 1.21
@@ -40,335 +40,337 @@
* DocumentCreator from the command-line arguments.
*/
public abstract class AbstractIndexer implements Indexer {
- private static Category log = Category.getInstance(AbstractIndexer.class);
-
- private DocumentCreator documentCreator;
- private Element indexer;
- private String configFileName;
-
- /**
- * Creates a new instance of AbstractIndexer
- */
- public AbstractIndexer() {
- }
-
- /**
- * Returns the DocumentCreator of this indexer.
- */
- protected DocumentCreator getDocumentCreator() {
- return documentCreator;
- }
-
- /**
- * Initializes this indexer with command-line parameters.
- */
- public void configure(Element indexer, String configFileName) throws Exception {
- documentCreator = createDocumentCreator(indexer, configFileName);
- this.indexer = indexer;
- this.configFileName = configFileName;
- }
-
- /**
- * DOCUMENT ME!
- *
- * @param element DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception DOCUMENT ME!
- */
- public abstract DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception;
-
- /**
- * Updates the index incrementally.
- * Walk directory hierarchy in uid order, while keeping uid iterator from
- * existing index in sync. Mismatches indicate one of:
- * <ol>
- * <li>old documents to be deleted</li>
- * <li>unchanged documents, to be left alone, or</li>
- * <li>new documents, to be indexed.</li>
- * </ol>
- */
- public void updateIndex(File dumpDirectory, File index) throws Exception {
- deleteStaleDocuments(dumpDirectory, index);
- doIndex(dumpDirectory, index, false);
- }
-
- /**
- * Updates the index re one document
- *
- * <ol>
- * <li>old documents to be deleted</li>
- * <li>unchanged documents, to be left alone, or</li>
- * <li>new documents, to be indexed.</li>
- * </ol>
- */
- public void indexDocument(File file) throws Exception {
- IndexConfiguration config = new IndexConfiguration(configFileName);
- log.debug("File: " + file);
-
- File dumpDir = new File(config.resolvePath(config.getHTDocsDumpDir()));
- log.debug("Dump dir: " + dumpDir);
-
- File indexDir = new File(config.resolvePath(config.getIndexDir()));
- log.debug("Index dir: " + indexDir);
-
- String id = IndexIterator.createID(file, dumpDir);
- log.debug("id: " + id);
-
- // Delete from index
- IndexReader reader = IndexReader.open(indexDir.getAbsolutePath());
- Term term = new Term("id", id);
- int numberOfDeletedDocuments = reader.delete(term);
- if (numberOfDeletedDocuments == 1) {
- log.error("Document has been deleted: " + term);
- } else {
- log.warn("No such document found in this index: " + term);
- }
- log.error("Number of delected documents: " + numberOfDeletedDocuments);
- log.error("Current number of documents in this index: " + reader.numDocs());
- reader.close();
-
- // Append to index
- Document doc = getDocumentCreator().getDocument(file, dumpDir);
- IndexWriter writer = new IndexWriter(indexDir, new StandardAnalyzer(), false);
- writer.maxFieldLength = 1000000;
- writer.addDocument(doc);
- writer.optimize();
- writer.close();
- }
-
- /**
- * Creates a new index.
- */
- public void createIndex(File dumpDirectory, File index)
- throws Exception {
- doIndex(dumpDirectory, index, true);
- }
-
- /**
- * Index files
- *
- * @param dumpDirectory Directory where the files to be indexed are located
- * @param index Directory where the index shall be located
- * @param create <strong>true</strong> means the index will be created from scratch, <strong>false</strong> means it will be indexed incrementally
- */
- public void doIndex(File dumpDirectory, File index, boolean create) {
- if (!index.isDirectory()) {
- index.mkdirs();
- log.warn("Directory has been created: " + index.getAbsolutePath());
- }
- try {
- IndexWriter writer = new IndexWriter(index.getAbsolutePath(), new StandardAnalyzer(), create);
- writer.maxFieldLength = 1000000;
-
- IndexInformation info = new IndexInformation(index.getAbsolutePath(), dumpDirectory, getFilter(indexer, configFileName), create);
-
- IndexHandler handler;
-
- if (create) {
- handler = new CreateIndexHandler(dumpDirectory, info, writer);
- } else {
- handler = new UpdateIndexHandler(dumpDirectory, info, writer);
- }
-
- IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
- iterator.addHandler(handler);
- iterator.iterate(dumpDirectory);
-
- writer.optimize();
- writer.close();
- } catch (IOException e) {
- log.error(e);
- }
- }
-
- /**
- * Delete the stale documents.
- */
- protected void deleteStaleDocuments(File dumpDirectory, File index)
- throws Exception {
- log.debug("Deleting stale documents");
-
- IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
- iterator.addHandler(new DeleteHandler());
- iterator.iterate(dumpDirectory);
- log.debug("Deleting stale documents finished");
- }
-
- /**
- * Returns the filter used to receive the indexable files. Might be overwritten by inherited class.
- */
- public FileFilter getFilter(Element indexer, String configFileName) {
- String[] indexableExtensions = { "html", "htm", "txt" };
- return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
- }
-
- /**
- * FileFilter used to obtain the files to index.
- */
- public class DefaultIndexFilter implements FileFilter {
- protected String[] indexableExtensions;
-
- /**
- * Default indexable extensions: html, htm, txt
- */
- public DefaultIndexFilter() {
- String[] iE = { "html", "htm", "txt" };
- indexableExtensions = iE;
- }
-
- /**
- *
- */
- public DefaultIndexFilter(String[] indexableExtensions) {
- this.indexableExtensions = indexableExtensions;
- }
-
- /** Tests whether or not the specified abstract pathname should be
- * included in a pathname list.
- *
- * @param pathname The abstract pathname to be tested
- * @return <code>true</code> if and only if <code>pathname</code> should be included
- *
- */
- public boolean accept(File file) {
- boolean accept;
-
- if (file.isDirectory()) {
- accept = true;
- } else {
- String fileName = file.getName();
- String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
- accept = Arrays.asList(indexableExtensions).contains(extension);
- }
-
- return accept;
- }
- }
-
- /**
- * Deletes all stale documents up to the document representing the next file.
- * The following documents are deleted:
- * <ul>
- * <li>representing files that where removed</li>
- * <li>representing the same file but are older than the current file</li>
- * </ul>
- */
- public class DeleteHandler extends AbstractIndexIteratorHandler {
- /** Handles a stale document.
- *
- */
- public void handleStaleDocument(IndexReader reader, Term term) {
- log.debug("deleting " +
- IndexIterator.uid2url(term.text()));
-
- try {
- int deletedDocuments = reader.delete(term);
- log.debug("deleted " + deletedDocuments +
- " documents.");
- } catch (IOException e) {
- log.error(e);
- }
- }
- }
-
- /**
- * DOCUMENT ME!
- */
- public class IndexHandler extends AbstractIndexIteratorHandler {
- /**
- * Creates a new IndexHandler object.
- *
- * @param dumpDirectory DOCUMENT ME!
- * @param info DOCUMENT ME!
- * @param writer DOCUMENT ME!
- */
- public IndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- this.info = info;
- this.dumpDirectory = dumpDirectory;
- this.writer = writer;
- }
-
- private IndexInformation info;
-
- protected IndexInformation getInformation() {
- return info;
- }
-
- private File dumpDirectory;
-
- protected File getDumpDirectory() {
- return dumpDirectory;
- }
-
- private IndexWriter writer;
-
- protected IndexWriter getWriter() {
- return writer;
- }
-
- /**
- * Add document to index
- */
- protected void addFile(File file) {
- log.debug("adding document: " + file.getAbsolutePath());
-
- try {
- Document doc = getDocumentCreator().getDocument(file, dumpDirectory);
- writer.addDocument(doc);
- } catch (Exception e) {
- log.error(e);
- }
-
- info.increase();
- log.info(info.printProgress());
- }
- }
-
- /**
- * DOCUMENT ME!
- */
- public class CreateIndexHandler extends IndexHandler {
- /**
- * Creates a new CreateIndexHandler object.
- *
- * @param dumpDirectory DOCUMENT ME!
- * @param info DOCUMENT ME!
- * @param writer DOCUMENT ME!
- */
- public CreateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- super(dumpDirectory, info, writer);
- }
-
- /**
- * Handles a file. Used when creating a new index.
- */
- public void handleFile(IndexReader reader, File file) {
- addFile(file);
- }
- }
-
- /**
- * DOCUMENT ME!
- */
- public class UpdateIndexHandler extends IndexHandler {
- /**
- * Creates a new UpdateIndexHandler object.
- *
- * @param dumpDirectory DOCUMENT ME!
- * @param info DOCUMENT ME!
- * @param writer DOCUMENT ME!
- */
- public UpdateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- super(dumpDirectory, info, writer);
- }
-
- /**
- * Handles a new document. Used when updating the index.
- */
- public void handleNewDocument(IndexReader reader, Term term, File file) {
- addFile(file);
- }
- }
+ private static Category log = Category.getInstance(AbstractIndexer.class);
+
+ private DocumentCreator documentCreator;
+ private Element indexer;
+ private String configFileName;
+
+ /**
+ * Creates a new instance of AbstractIndexer
+ */
+ public AbstractIndexer() {
+ }
+
+ /**
+ * Returns the DocumentCreator of this indexer.
+ */
+ protected DocumentCreator getDocumentCreator() {
+ return documentCreator;
+ }
+
+ /**
+ * Initializes this indexer with command-line parameters.
+ */
+ public void configure(Element indexer, String configFileName) throws Exception {
+ documentCreator = createDocumentCreator(indexer, configFileName);
+ this.indexer = indexer;
+ this.configFileName = configFileName;
+ }
+
+ /**
+ * DOCUMENT ME!
+ *
+ * @param element DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception DOCUMENT ME!
+ */
+ public abstract DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception;
+
+ /**
+ * Updates the index incrementally.
+ * Walk directory hierarchy in uid order, while keeping uid iterator from
+ * existing index in sync. Mismatches indicate one of:
+ * <ol>
+ * <li>old documents to be deleted</li>
+ * <li>unchanged documents, to be left alone, or</li>
+ * <li>new documents, to be indexed.</li>
+ * </ol>
+ */
+ public void updateIndex(File dumpDirectory, File index) throws Exception {
+ deleteStaleDocuments(dumpDirectory, index);
+ doIndex(dumpDirectory, index, false);
+ }
+
+ /**
+ * Updates the index re one document
+ *
+ * <ol>
+ * <li>old documents to be deleted</li>
+ * <li>unchanged documents, to be left alone, or</li>
+ * <li>new documents, to be indexed.</li>
+ * </ol>
+ */
+ public void indexDocument(File file) throws Exception {
+ IndexConfiguration config = new IndexConfiguration(configFileName);
+ log.debug("File: " + file);
+
+ File dumpDir = new File(config.resolvePath(config.getHTDocsDumpDir()));
+ log.debug("Dump dir: " + dumpDir);
+
+ File indexDir = new File(config.resolvePath(config.getIndexDir()));
+ log.debug("Index dir: " + indexDir);
+
+ String id = IndexIterator.createID(file, dumpDir);
+
+ // Delete from index
+ IndexReader reader = IndexReader.open(indexDir.getAbsolutePath());
+ Term term = new Term("id", id);
+ log.debug(term.toString());
+ int numberOfDeletedDocuments = reader.delete(term);
+ if (numberOfDeletedDocuments == 1) {
+ log.info("Document has been deleted: " + term);
+ } else {
+ log.warn("No such document found in this index: " + term);
+ }
+ //log.debug("Number of deleted documents: " + numberOfDeletedDocuments);
+ //log.debug("Current number of documents in this index: " + reader.numDocs());
+ reader.close();
+
+ // Append to index
+ Document doc = getDocumentCreator().getDocument(new File(dumpDir, id), dumpDir);
+ IndexWriter writer = new IndexWriter(indexDir, new StandardAnalyzer(), false);
+ writer.maxFieldLength = 1000000;
+ writer.addDocument(doc);
+ //log.debug("Document has been added: " + doc);
+ log.info("Document has been added: " + id);
+ writer.optimize();
+ writer.close();
+ }
+
+ /**
+ * Creates a new index.
+ */
+ public void createIndex(File dumpDirectory, File index)
+ throws Exception {
+ doIndex(dumpDirectory, index, true);
+ }
+
+ /**
+ * Index files
+ *
+ * @param dumpDirectory Directory where the files to be indexed are located
+ * @param index Directory where the index shall be located
+ * @param create <strong>true</strong> means the index will be created from scratch, <strong>false</strong> means it will be indexed incrementally
+ */
+ public void doIndex(File dumpDirectory, File index, boolean create) {
+ if (!index.isDirectory()) {
+ index.mkdirs();
+ log.warn("Directory has been created: " + index.getAbsolutePath());
+ }
+ try {
+ IndexWriter writer = new IndexWriter(index.getAbsolutePath(), new StandardAnalyzer(), create);
+ writer.maxFieldLength = 1000000;
+
+ IndexInformation info = new IndexInformation(index.getAbsolutePath(), dumpDirectory, getFilter(indexer, configFileName), create);
+
+ IndexHandler handler;
+
+ if (create) {
+ handler = new CreateIndexHandler(dumpDirectory, info, writer);
+ } else {
+ handler = new UpdateIndexHandler(dumpDirectory, info, writer);
+ }
+
+ IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
+ iterator.addHandler(handler);
+ iterator.iterate(dumpDirectory);
+
+ writer.optimize();
+ writer.close();
+ } catch (IOException e) {
+ log.error(e);
+ }
+ }
+
+ /**
+ * Delete the stale documents.
+ */
+ protected void deleteStaleDocuments(File dumpDirectory, File index)
+ throws Exception {
+ log.debug("Deleting stale documents");
+
+ IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
+ iterator.addHandler(new DeleteHandler());
+ iterator.iterate(dumpDirectory);
+ log.debug("Deleting stale documents finished");
+ }
+
+ /**
+ * Returns the filter used to receive the indexable files. Might be overwritten by inherited class.
+ */
+ public FileFilter getFilter(Element indexer, String configFileName) {
+ String[] indexableExtensions = { "html", "htm", "txt" };
+ return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
+ }
+
+ /**
+ * FileFilter used to obtain the files to index.
+ */
+ public class DefaultIndexFilter implements FileFilter {
+ protected String[] indexableExtensions;
+
+ /**
+ * Default indexable extensions: html, htm, txt
+ */
+ public DefaultIndexFilter() {
+ String[] iE = { "html", "htm", "txt" };
+ indexableExtensions = iE;
+ }
+
+ /**
+ *
+ */
+ public DefaultIndexFilter(String[] indexableExtensions) {
+ this.indexableExtensions = indexableExtensions;
+ }
+
+ /** Tests whether or not the specified abstract pathname should be
+ * included in a pathname list.
+ *
+ * @param pathname The abstract pathname to be tested
+ * @return <code>true</code> if and only if <code>pathname</code> should be included
+ *
+ */
+ public boolean accept(File file) {
+ boolean accept;
+
+ if (file.isDirectory()) {
+ accept = true;
+ } else {
+ String fileName = file.getName();
+ String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
+ accept = Arrays.asList(indexableExtensions).contains(extension);
+ }
+
+ return accept;
+ }
+ }
+
+ /**
+ * Deletes all stale documents up to the document representing the next file.
+ * The following documents are deleted:
+ * <ul>
+ * <li>representing files that where removed</li>
+ * <li>representing the same file but are older than the current file</li>
+ * </ul>
+ */
+ public class DeleteHandler extends AbstractIndexIteratorHandler {
+ /** Handles a stale document.
+ *
+ */
+ public void handleStaleDocument(IndexReader reader, Term term) {
+ log.debug("deleting " +
+ IndexIterator.uid2url(term.text()));
+
+ try {
+ int deletedDocuments = reader.delete(term);
+ log.debug("deleted " + deletedDocuments +
+ " documents.");
+ } catch (IOException e) {
+ log.error(e);
+ }
+ }
+ }
+
+ /**
+ * DOCUMENT ME!
+ */
+ public class IndexHandler extends AbstractIndexIteratorHandler {
+ /**
+ * Creates a new IndexHandler object.
+ *
+ * @param dumpDirectory DOCUMENT ME!
+ * @param info DOCUMENT ME!
+ * @param writer DOCUMENT ME!
+ */
+ public IndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ this.info = info;
+ this.dumpDirectory = dumpDirectory;
+ this.writer = writer;
+ }
+
+ private IndexInformation info;
+
+ protected IndexInformation getInformation() {
+ return info;
+ }
+
+ private File dumpDirectory;
+
+ protected File getDumpDirectory() {
+ return dumpDirectory;
+ }
+
+ private IndexWriter writer;
+
+ protected IndexWriter getWriter() {
+ return writer;
+ }
+
+ /**
+ * Add document to index
+ */
+ protected void addFile(File file) {
+ log.debug("adding document: " + file.getAbsolutePath());
+
+ try {
+ Document doc = getDocumentCreator().getDocument(file, dumpDirectory);
+ writer.addDocument(doc);
+ } catch (Exception e) {
+ log.error(e);
+ }
+
+ info.increase();
+ log.info(info.printProgress());
+ }
+ }
+
+ /**
+ * DOCUMENT ME!
+ */
+ public class CreateIndexHandler extends IndexHandler {
+ /**
+ * Creates a new CreateIndexHandler object.
+ *
+ * @param dumpDirectory DOCUMENT ME!
+ * @param info DOCUMENT ME!
+ * @param writer DOCUMENT ME!
+ */
+ public CreateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ super(dumpDirectory, info, writer);
+ }
+
+ /**
+ * Handles a file. Used when creating a new index.
+ */
+ public void handleFile(IndexReader reader, File file) {
+ addFile(file);
+ }
+ }
+
+ /**
+ * DOCUMENT ME!
+ */
+ public class UpdateIndexHandler extends IndexHandler {
+ /**
+ * Creates a new UpdateIndexHandler object.
+ *
+ * @param dumpDirectory DOCUMENT ME!
+ * @param info DOCUMENT ME!
+ * @param writer DOCUMENT ME!
+ */
+ public UpdateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ super(dumpDirectory, info, writer);
+ }
+
+ /**
+ * Handles a new document. Used when updating the index.
+ */
+ public void handleNewDocument(IndexReader reader, Term term, File file) {
+ addFile(file);
+ }
+ }
}
1.11 +63 -63 cocoon-lenya/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java
Index: AbstractDocumentCreator.java
===================================================================
RCS file: /home/cvs/cocoon-lenya/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -r1.10 -r1.11
--- AbstractDocumentCreator.java 1 Aug 2004 10:48:05 -0000 1.10
+++ AbstractDocumentCreator.java 2 Aug 2004 00:19:42 -0000 1.11
@@ -28,66 +28,66 @@
import org.apache.log4j.Category;
public class AbstractDocumentCreator implements DocumentCreator {
- Category log = Category.getInstance(AbstractDocumentCreator.class);
-
- /** Creates a new instance of AbstractDocumentCreator */
- public AbstractDocumentCreator() {
- }
-
- /**
- * DOCUMENT ME!
- *
- * @param file DOCUMENT ME!
- * @param htdocsDumpDir DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception DOCUMENT ME!
- */
- public Document getDocument(File file, File htdocsDumpDir)
- throws Exception {
- // make a new, empty document
- Document doc = new Document();
-
- // Add the url as a field named "url". Use an UnIndexed field, so
- // that the url is just stored with the document, but is not searchable.
- String requestURI = file.getPath().replace(File.separatorChar, '/').substring(htdocsDumpDir.getPath()
- .length());
- if (requestURI.substring(requestURI.length() - 8).equals(".pdf.txt")) {
- requestURI = requestURI.substring(0, requestURI.length() - 4); // Remove .txt extension from PDF text file
- }
-
- doc.add(Field.UnIndexed("url", requestURI));
-
- // Add the mime-type as a field named "mime-type"
- if (requestURI.substring(requestURI.length() - 5).equals(".html")) {
- doc.add(Field.UnIndexed("mime-type", "text/html"));
- } else if (requestURI.substring(requestURI.length() - 4).equals(".txt")) {
- doc.add(Field.UnIndexed("mime-type", "text/plain"));
- } else if (requestURI.substring(requestURI.length() - 4).equals(".pdf")) {
- doc.add(Field.UnIndexed("mime-type", "application/pdf"));
- } else {
- // Don't add any mime-type field
- //doc.add(Field.UnIndexed("mime-type", "null"));
- }
-
- // Add the last modified date of the file a field named "modified". Use a
- // Keyword field, so that it's searchable, but so that no attempt is made
- // to tokenize the field into words.
- doc.add(Field.Keyword("modified", DateField.timeToString(file.lastModified())));
-
- // Add the id as a field, so that index can be incrementally maintained.
- String id = IndexIterator.createID(file, htdocsDumpDir);
- log.debug(id);
- doc.add(Field.Keyword("id", id));
-
- // Add the uid as a field, so that index can be incrementally maintained.
- // This field is not stored with document, it is indexed, but it is not
- // tokenized prior to indexing.
- String uid = IndexIterator.createUID(file, htdocsDumpDir);
- log.debug(uid);
- doc.add(new Field("uid", uid, false, true, false));
-
- return doc;
- }
+ Category log = Category.getInstance(AbstractDocumentCreator.class);
+
+ /** Creates a new instance of AbstractDocumentCreator */
+ public AbstractDocumentCreator() {
+ }
+
+ /**
+ * DOCUMENT ME!
+ *
+ * @param file DOCUMENT ME!
+ * @param htdocsDumpDir DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception DOCUMENT ME!
+ */
+ public Document getDocument(File file, File htdocsDumpDir)
+ throws Exception {
+ // make a new, empty document
+ Document doc = new Document();
+
+ // Add the url as a field named "url". Use an UnIndexed field, so
+ // that the url is just stored with the document, but is not searchable.
+ String requestURI = file.getPath().replace(File.separatorChar, '/').substring(htdocsDumpDir.getPath()
+ .length());
+ if (requestURI.substring(requestURI.length() - 8).equals(".pdf.txt")) {
+ requestURI = requestURI.substring(0, requestURI.length() - 4); // Remove .txt extension from PDF text file
+ }
+
+ doc.add(Field.UnIndexed("url", requestURI));
+
+ // Add the mime-type as a field named "mime-type"
+ if (requestURI.substring(requestURI.length() - 5).equals(".html")) {
+ doc.add(Field.UnIndexed("mime-type", "text/html"));
+ } else if (requestURI.substring(requestURI.length() - 4).equals(".txt")) {
+ doc.add(Field.UnIndexed("mime-type", "text/plain"));
+ } else if (requestURI.substring(requestURI.length() - 4).equals(".pdf")) {
+ doc.add(Field.UnIndexed("mime-type", "application/pdf"));
+ } else {
+ // Don't add any mime-type field
+ //doc.add(Field.UnIndexed("mime-type", "null"));
+ }
+
+ // Add the last modified date of the file a field named "modified". Use a
+ // Keyword field, so that it's searchable, but so that no attempt is made
+ // to tokenize the field into words.
+ doc.add(Field.Keyword("modified", DateField.timeToString(file.lastModified())));
+
+ // Add the id as a field, so that index can be incrementally maintained.
+ String id = IndexIterator.createID(file, htdocsDumpDir);
+ log.debug(id);
+ doc.add(Field.Keyword("id", id));
+
+ // Add the uid as a field, so that index can be incrementally maintained.
+ // This field is not stored with document, it is indexed, but it is not
+ // tokenized prior to indexing.
+ String uid = IndexIterator.createUID(file, htdocsDumpDir);
+ log.debug(uid);
+ doc.add(new Field("uid", uid, false, true, false));
+
+ return doc;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: lenya-cvs-unsubscribe@cocoon.apache.org
For additional commands, e-mail: lenya-cvs-help@cocoon.apache.org