You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2010/06/30 12:36:29 UTC
svn commit: r959259 [5/12] - in /nutch/branches/nutchbase: ./ bin/ conf/
contrib/ docs/ ivy/ lib/ lib/jetty-ext/ src/engines/ src/gora/ src/java/
src/java/org/apache/nutch/analysis/ src/java/org/apache/nutch/clustering/
src/java/org/apache/nutch/crawl/...
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerJob.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerJob.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerJob.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,111 @@
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+
+import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.Tool;
+import org.apache.nutch.crawl.GeneratorJob;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.parse.ParseStatusCodes;
+import org.apache.nutch.parse.ParseStatusUtils;
+import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.storage.ParseStatus;
+import org.apache.nutch.storage.StorageUtils;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.TableUtil;
+import org.gora.mapreduce.GoraMapper;
+import org.gora.mapreduce.StringComparator;
+
+public abstract class IndexerJob
+extends GoraMapper<String, WebPage, String, WebPage>
+implements Tool {
+
+ public static final Log LOG = LogFactory.getLog(IndexerJob.class);
+
+ private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
+ private static final Utf8 REINDEX = new Utf8("-reindex");
+
+ private Configuration conf;
+
+ protected Utf8 crawlId;
+
+ static {
+ FIELDS.add(WebPage.Field.SIGNATURE);
+ FIELDS.add(WebPage.Field.PARSE_STATUS);
+ FIELDS.add(WebPage.Field.SCORE);
+ FIELDS.add(WebPage.Field.MARKERS);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public void setup(Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ crawlId = new Utf8(conf.get(GeneratorJob.CRAWL_ID, Nutch.ALL_CRAWL_ID_STR));
+ }
+
+ @Override
+ public void map(String key, WebPage page, Context context)
+ throws IOException, InterruptedException {
+ ParseStatus pstatus = page.getParseStatus();
+ if (pstatus == null || !ParseStatusUtils.isSuccess(pstatus)
+ || pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) {
+ return; // filter urls not parsed
+ }
+
+ Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page);
+ if (!crawlId.equals(REINDEX)) {
+ if (!NutchJob.shouldProcess(mark, crawlId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping " + TableUtil.unreverseUrl(key) + "; different crawl id");
+ }
+ return;
+ }
+ }
+
+ context.write(key, page);
+ }
+
+ private static Collection<WebPage.Field> getFields(Job job) {
+ Configuration conf = job.getConfiguration();
+ Collection<WebPage.Field> columns = new HashSet<WebPage.Field>(FIELDS);
+ IndexingFilters filters = new IndexingFilters(conf);
+ columns.addAll(filters.getFields());
+ ScoringFilters scoringFilters = new ScoringFilters(conf);
+ columns.addAll(scoringFilters.getFields());
+ return columns;
+ }
+
+ protected Job createIndexJob(Configuration conf, String jobName, String crawlId)
+ throws IOException, ClassNotFoundException {
+ conf.set(GeneratorJob.CRAWL_ID, crawlId);
+ Job job = new NutchJob(conf, jobName);
+ // TODO: Figure out why this needs to be here
+ job.getConfiguration().setClass("mapred.output.key.comparator.class",
+ StringComparator.class, RawComparator.class);
+
+ Collection<WebPage.Field> fields = getFields(job);
+ StorageUtils.initMapperJob(job, fields, String.class, WebPage.class, this.getClass());
+ job.setReducerClass(IndexerReducer.class);
+ job.setOutputFormatClass(IndexerOutputFormat.class);
+ return job;
+ }
+}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java Wed Jun 30 10:36:20 2010
@@ -18,17 +18,15 @@ package org.apache.nutch.indexer;
import java.io.IOException;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.nutch.indexer.NutchDocument;
public class IndexerOutputFormat
-extends FileOutputFormat<WritableComparable<?>, NutchDocument> {
+extends FileOutputFormat<String, NutchDocument> {
@Override
- public RecordWriter<WritableComparable<?>, NutchDocument> getRecordWriter(
+ public RecordWriter<String, NutchDocument> getRecordWriter(
TaskAttemptContext job) throws IOException, InterruptedException {
final NutchIndexWriter[] writers =
@@ -38,10 +36,10 @@ extends FileOutputFormat<WritableCompara
writer.open(job, FileOutputFormat.getUniqueFile(job, "part", ""));
}
- return new RecordWriter<WritableComparable<?>, NutchDocument>() {
+ return new RecordWriter<String, NutchDocument>() {
@Override
- public void write(WritableComparable<?> key, NutchDocument doc) throws IOException {
+ public void write(String key, NutchDocument doc) throws IOException {
for (final NutchIndexWriter writer : writers) {
writer.write(doc);
}
@@ -52,7 +50,7 @@ extends FileOutputFormat<WritableCompara
InterruptedException {
for (final NutchIndexWriter writer : writers) {
writer.close();
- }
+ }
}
};
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerReducer.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerReducer.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerReducer.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerReducer.java Wed Jun 30 10:36:20 2010
@@ -2,57 +2,59 @@ package org.apache.nutch.indexer;
import java.io.IOException;
+import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.nutch.indexer.IndexingException;
-import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.storage.StorageUtils;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.StringUtil;
-import org.apache.nutch.util.hbase.WebTableRow;
-import org.apache.nutch.util.hbase.TableUtil;
+import org.apache.nutch.util.TableUtil;
+import org.gora.store.DataStore;
public class IndexerReducer
-extends Reducer<ImmutableBytesWritable, WebTableRow, ImmutableBytesWritable, NutchDocument> {
+extends Reducer<String, WebPage, String, NutchDocument> {
+
+ public static final Log LOG = IndexerJob.LOG;
- public static final Log LOG = Indexer.LOG;
-
private IndexingFilters filters;
-
+
private ScoringFilters scoringFilters;
-
- private HTable table;
-
+
+ private DataStore<String, WebPage> store;
+
@Override
protected void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
filters = new IndexingFilters(conf);
- table = new HTable(conf.get(TableInputFormat.INPUT_TABLE));
scoringFilters = new ScoringFilters(conf);
+ try {
+ store = StorageUtils.createDataStore(conf, String.class, WebPage.class);
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
}
-
+
@Override
- protected void reduce(ImmutableBytesWritable key, Iterable<WebTableRow> values,
+ protected void reduce(String key, Iterable<WebPage> values,
Context context) throws IOException, InterruptedException {
- WebTableRow row = values.iterator().next();
+ WebPage page = values.iterator().next();
NutchDocument doc = new NutchDocument();
- doc.add("id", Bytes.toString(key.get()));
- doc.add("digest", StringUtil.toHexString(row.getSignature()));
+ doc.add("id", key);
+ doc.add("digest", StringUtil.toHexString(page.getSignature().array()));
- String url = TableUtil.unreverseUrl(Bytes.toString(key.get()));
+ String url = TableUtil.unreverseUrl(key);
if (LOG.isDebugEnabled()) {
LOG.debug("Indexing URL: " + url);
}
try {
- doc = filters.filter(doc, url, row);
+ doc = filters.filter(doc, url, page);
} catch (IndexingException e) {
LOG.warn("Error indexing "+key+": "+e);
return;
@@ -64,7 +66,7 @@ extends Reducer<ImmutableBytesWritable,
float boost = 1.0f;
// run scoring filters
try {
- boost = scoringFilters.indexerScore(url, doc, row, boost);
+ boost = scoringFilters.indexerScore(url, doc, page, boost);
} catch (final ScoringFilterException e) {
LOG.warn("Error calculating score " + key + ": " + e);
return;
@@ -74,14 +76,17 @@ extends Reducer<ImmutableBytesWritable,
// store boost for use by explain and dedup
doc.add("boost", Float.toString(boost));
- row.putMeta(Indexer.INDEX_MARK, TableUtil.YES_VAL);
- row.makeRowMutation().commit(table);
+ Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page);
+ if (mark != null) {
+ Mark.INDEX_MARK.putMark(page, Mark.UPDATEDB_MARK.checkMark(page));
+ store.put(key, page);
+ }
context.write(key, doc);
}
-
+
@Override
public void cleanup(Context context) throws IOException {
- table.close();
+ store.close();
}
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilter.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilter.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilter.java Wed Jun 30 10:36:20 2010
@@ -20,36 +20,32 @@ package org.apache.nutch.indexer;
// Hadoop imports
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-
-// Nutch imports
-import org.apache.nutch.indexer.IndexingException;
-import org.apache.nutch.indexer.NutchDocument;
-import org.apache.nutch.plugin.TablePluggable;
-import org.apache.nutch.util.hbase.WebTableRow;
+import org.apache.nutch.plugin.FieldPluggable;
+import org.apache.nutch.storage.WebPage;
/** Extension point for indexing. Permits one to add metadata to the indexed
* fields. All plugins found which implement this extension point are run
* sequentially on the parse.
*/
-public interface IndexingFilter extends TablePluggable, Configurable {
+public interface IndexingFilter extends FieldPluggable, Configurable {
/** The name of the extension point. */
final static String X_POINT_ID = IndexingFilter.class.getName();
/**
* Adds fields or otherwise modifies the document that will be indexed for a
* parse. Unwanted documents can be removed from indexing by returning a null value.
- *
+ *
* @param doc document instance for collecting fields
* @param url page url
- * @param url's row
+ * @param page
* @return modified (or a new) document instance, or null (meaning the document
* should be discarded)
* @throws IndexingException
*/
- NutchDocument filter(NutchDocument doc, String url, WebTableRow row)
+ NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException;
-
+
/** Adds index-level configuraition options.
* Implementations can update given configuration to pass document-independent
* information to indexing backends. As a rule of thumb, prefix meta keys
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilters.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilters.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilters.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilters.java Wed Jun 30 10:36:20 2010
@@ -22,17 +22,15 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-// Commons Logging imports
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import org.apache.nutch.plugin.*;
-import org.apache.nutch.util.ObjectCache;
-import org.apache.nutch.util.hbase.HbaseColumn;
-import org.apache.nutch.util.hbase.WebTableRow;
import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.indexer.IndexingException;
-import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.ExtensionPoint;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.ObjectCache;
/** Creates and caches {@link IndexingFilter} implementing plugins.*/
public class IndexingFilters {
@@ -103,12 +101,12 @@ public class IndexingFilters {
this.indexingFilters = (IndexingFilter[]) objectCache
.getObject(IndexingFilter.class.getName());
}
- }
+ }
/** Run all defined filters. */
- public NutchDocument filter(NutchDocument doc, String url, WebTableRow row)
+ public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
for (IndexingFilter indexingFilter : indexingFilters) {
- doc = indexingFilter.filter(doc, url, row);
+ doc = indexingFilter.filter(doc, url, page);
// break the loop if an indexing filter discards the doc
if (doc == null) return null;
}
@@ -116,12 +114,15 @@ public class IndexingFilters {
return doc;
}
- public Collection<HbaseColumn> getColumns() {
- Collection<HbaseColumn> columns = new HashSet<HbaseColumn>();
+ public Collection<WebPage.Field> getFields() {
+ Collection<WebPage.Field> columns = new HashSet<WebPage.Field>();
for (IndexingFilter indexingFilter : indexingFilters) {
- columns.addAll(indexingFilter.getColumns());
+ Collection<WebPage.Field> fields = indexingFilter.getFields();
+ if (fields != null) {
+ columns.addAll(fields);
+ }
}
- return columns;
+ return columns;
}
-
+
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java Wed Jun 30 10:36:20 2010
@@ -24,7 +24,6 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.nutch.indexer.DeleteDuplicates;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
@@ -60,7 +59,7 @@ import org.apache.solr.common.SolrDocume
* </li>
* </ul>
*
- * Note that unlike {@link DeleteDuplicates} we assume that two documents in
+ * Note that we assume that two documents in
* a solr index will never have the same URL. So this class only deals with
* documents with <b>different</b> URLs but the same digest.
*/
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexerJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexerJob.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexerJob.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexerJob.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.solr;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.indexer.IndexerJob;
+import org.apache.nutch.indexer.NutchIndexWriterFactory;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+
+public class SolrIndexerJob extends IndexerJob {
+
+ public static Log LOG = LogFactory.getLog(SolrIndexerJob.class);
+
+ private void indexSolr(String solrUrl, String crawlId) throws Exception {
+ LOG.info("SolrIndexerJob: starting");
+
+ NutchIndexWriterFactory.addClassToConf(getConf(), SolrWriter.class);
+ getConf().set(SolrConstants.SERVER_URL, solrUrl);
+
+ Job job = createIndexJob(getConf(), "solr-index", crawlId);
+ Path tmp = new Path("tmp_" + System.currentTimeMillis() + "-"
+ + new Random().nextInt());
+
+ FileOutputFormat.setOutputPath(job, tmp);
+ boolean success = false;
+ try {
+ success = job.waitForCompletion(true);
+ // do the commits once and for all the reducers in one go
+ SolrServer solr = new CommonsHttpSolrServer(solrUrl);
+ solr.commit();
+ } finally {
+ FileSystem.get(getConf()).delete(tmp, true);
+ }
+ LOG.info("SolrIndexerJob: " + (success ? "done" : "failed"));
+ }
+
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println("Usage: SolrIndexerJob <solr url> (<crawl id> | -all | -reindex)");
+ return -1;
+ }
+
+ try {
+ indexSolr(args[0], args[1]);
+ return 0;
+ } catch (final Exception e) {
+ LOG.fatal("SolrIndexerJob: " + StringUtils.stringifyException(e));
+ return -1;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ final int res = ToolRunner.run(NutchConfiguration.create(),
+ new SolrIndexerJob(), args);
+ System.exit(res);
+ }
+}
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.solr;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.ObjectCache;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+public class SolrMappingReader {
+ public static Log LOG = LogFactory.getLog(SolrMappingReader.class);
+
+ /** The property name of the parse solr index mapping location */
+ private static final String SS_FILE_MAPPING = "solrindex.mapping.file";
+
+ private Configuration conf;
+
+ private Map<String, String> keyMap = new HashMap<String, String>();
+ private Map<String, String> copyMap = new HashMap<String, String>();
+ private String uniqueKey = "id";
+
+ public static synchronized SolrMappingReader getInstance(Configuration conf) {
+ ObjectCache cache = ObjectCache.get(conf);
+ SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
+ if (instance == null) {
+ instance = new SolrMappingReader(conf);
+ cache.setObject(SolrMappingReader.class.getName(), instance);
+ }
+ return instance;
+ }
+
+ protected SolrMappingReader(Configuration conf) {
+ this.conf = conf;
+ parseMapping();
+ }
+
+ private void parseMapping() {
+ InputStream ssInputStream = null;
+ ssInputStream = conf.getConfResourceAsInputStream(conf.get(SS_FILE_MAPPING, "solrindex-mapping.xml"));
+ InputSource inputSource = new InputSource(ssInputStream);
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document document = builder.parse(inputSource);
+ Element rootElement = document.getDocumentElement();
+ NodeList fieldList = rootElement.getElementsByTagName("field");
+ if (fieldList.getLength() > 0) {
+ for (int i = 0; i < fieldList.getLength(); i++) {
+ Element element = (Element) fieldList.item(i);
+ LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+ keyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+ }
+ }
+ NodeList copyFieldList = rootElement.getElementsByTagName("copyField");
+ if (copyFieldList.getLength() > 0) {
+ for (int i = 0; i < copyFieldList.getLength(); i++) {
+ Element element = (Element) copyFieldList.item(i);
+ LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+ copyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+ }
+ }
+ NodeList uniqueKeyItem = rootElement.getElementsByTagName("uniqueKey");
+ if (uniqueKeyItem.getLength() > 1) {
+ LOG.warn("More than one unique key definitions found in solr index mapping, using default 'id'");
+ uniqueKey = "id";
+ }
+ else if (uniqueKeyItem.getLength() == 0) {
+ LOG.warn("No unique key definition found in solr index mapping using, default 'id'");
+ }
+ else{
+ uniqueKey = uniqueKeyItem.item(0).getFirstChild().getNodeValue();
+ }
+ } catch (MalformedURLException e) {
+ LOG.warn(e.toString());
+ } catch (SAXException e) {
+ LOG.warn(e.toString());
+ } catch (IOException e) {
+ LOG.warn(e.toString());
+ } catch (ParserConfigurationException e) {
+ LOG.warn(e.toString());
+ }
+ }
+
+ public Map<String, String> getKeyMap() {
+ return keyMap;
+ }
+
+ public Map<String, String> getCopyMap() {
+ return copyMap;
+ }
+
+ public String getUniqueKey() {
+ return uniqueKey;
+ }
+
+ public String hasCopy(String key) {
+ if (copyMap.containsKey(key)) {
+ key = (String) copyMap.get(key);
+ }
+ return key;
+ }
+
+ public String mapKey(String key) throws IOException {
+ if(keyMap.containsKey(key)) {
+ key = (String) keyMap.get(key);
+ }
+ return key;
+ }
+
+ public String mapCopyKey(String key) throws IOException {
+ if(copyMap.containsKey(key)) {
+ key = (String) copyMap.get(key);
+ }
+ return key;
+ }
+}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrWriter.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrWriter.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrWriter.java Wed Jun 30 10:36:20 2010
@@ -33,6 +33,7 @@ import org.apache.solr.common.SolrInputD
public class SolrWriter implements NutchIndexWriter {
private SolrServer solr;
+ private SolrMappingReader solrMapping;
private final List<SolrInputDocument> inputDocs =
new ArrayList<SolrInputDocument>();
@@ -45,6 +46,7 @@ public class SolrWriter implements Nutch
Configuration conf = job.getConfiguration();
solr = new CommonsHttpSolrServer(conf.get(SolrConstants.SERVER_URL));
commitSize = conf.getInt(SolrConstants.COMMIT_SIZE, 1000);
+ solrMapping = SolrMappingReader.getInstance(conf);
}
@Override
@@ -52,7 +54,11 @@ public class SolrWriter implements Nutch
final SolrInputDocument inputDoc = new SolrInputDocument();
for(final Entry<String, List<String>> e : doc) {
for (final String val : e.getValue()) {
- inputDoc.addField(e.getKey(), val);
+ inputDoc.addField(solrMapping.mapKey(e.getKey()), val);
+ String sCopy = solrMapping.mapCopyKey(e.getKey());
+ if (sCopy != e.getKey()) {
+ inputDoc.addField(sCopy, val);
+ }
}
}
inputDoc.setDocumentBoost(doc.getScore());
@@ -74,7 +80,6 @@ public class SolrWriter implements Nutch
solr.add(inputDocs);
inputDocs.clear();
}
- solr.commit();
} catch (final SolrServerException e) {
throw new IOException(e);
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/HttpHeaders.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/HttpHeaders.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/HttpHeaders.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/HttpHeaders.java Wed Jun 30 10:36:20 2010
@@ -17,6 +17,7 @@
package org.apache.nutch.metadata;
+
/**
* A collection of HTTP header names.
*
@@ -29,21 +30,21 @@ package org.apache.nutch.metadata;
public interface HttpHeaders {
public final static String CONTENT_ENCODING = "Content-Encoding";
-
+
public final static String CONTENT_LANGUAGE = "Content-Language";
public final static String CONTENT_LENGTH = "Content-Length";
-
+
public final static String CONTENT_LOCATION = "Content-Location";
-
+
public static final String CONTENT_DISPOSITION = "Content-Disposition";
public final static String CONTENT_MD5 = "Content-MD5";
-
+
public final static String CONTENT_TYPE = "Content-Type";
-
+
public final static String LAST_MODIFIED = "Last-Modified";
-
+
public final static String LOCATION = "Location";
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/Nutch.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/Nutch.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/Nutch.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/Nutch.java Wed Jun 30 10:36:20 2010
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.metadata;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.io.Text;
@@ -26,10 +27,10 @@ import org.apache.hadoop.io.Text;
* @author Jérôme Charron
*/
public interface Nutch {
-
+
public static final String ORIGINAL_CHAR_ENCODING =
"OriginalCharEncoding";
-
+
public static final String CHAR_ENCODING_FOR_CONVERSION =
"CharEncodingForConversion";
@@ -46,14 +47,16 @@ public interface Nutch {
public static final String PROTO_STATUS_KEY = "_pst_";
public static final Text WRITABLE_PROTO_STATUS_KEY = new Text(PROTO_STATUS_KEY);
-
+
public static final String FETCH_TIME_KEY = "_ftk_";
-
+
public static final String FETCH_STATUS_KEY = "_fst_";
/** Sites may request that search engines don't provide access to cached documents. */
public static final String CACHING_FORBIDDEN_KEY = "caching.forbidden";
+ public static final Utf8 CACHING_FORBIDDEN_KEY_UTF8 = new Utf8(CACHING_FORBIDDEN_KEY);
+
/** Show both original forbidden content and summaries (default). */
public static final String CACHING_FORBIDDEN_NONE = "none";
@@ -66,4 +69,8 @@ public interface Nutch {
public static final String REPR_URL_KEY = "_repr_";
public static final Text WRITABLE_REPR_URL_KEY = new Text(REPR_URL_KEY);
+
+ public static final String ALL_CRAWL_ID_STR = "-all";
+
+ public static final Utf8 ALL_CRAWL_ID = new Utf8(ALL_CRAWL_ID_STR);
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/net/URLNormalizers.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/net/URLNormalizers.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/net/URLNormalizers.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/net/URLNormalizers.java Wed Jun 30 10:36:20 2010
@@ -79,11 +79,11 @@ public final class URLNormalizers {
* this scope will be used.
*/
public static final String SCOPE_DEFAULT = "default";
- /** Scope used by {@link org.apache.nutch.crawl.PartitionUrlByHost}. */
+ /** Scope used by {@link org.apache.nutch.crawl.URLPartitioner}. */
public static final String SCOPE_PARTITION = "partition";
- /** Scope used by {@link org.apache.nutch.crawl.Generator}. */
+ /** Scope used by {@link org.apache.nutch.crawl.GeneratorJob}. */
public static final String SCOPE_GENERATE_HOST_COUNT = "generate_host_count";
- /** Scope used by {@link org.apache.nutch.fetcher.Fetcher} when processing
+ /** Scope used by {@link org.apache.nutch.fetcher.FetcherJob} when processing
* redirect URLs.
*/
public static final String SCOPE_FETCHER = "fetcher";
@@ -91,7 +91,7 @@ public final class URLNormalizers {
public static final String SCOPE_CRAWLDB = "crawldb";
/** Scope used when updating the LinkDb with new URLs. */
public static final String SCOPE_LINKDB = "linkdb";
- /** Scope used by {@link org.apache.nutch.crawl.Injector}. */
+ /** Scope used by {@link org.apache.nutch.crawl.InjectorJob}. */
public static final String SCOPE_INJECT = "inject";
/** Scope used when constructing new {@link org.apache.nutch.parse.Outlink} instances. */
public static final String SCOPE_OUTLINK = "outlink";
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilter.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilter.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilter.java Wed Jun 30 10:36:20 2010
@@ -1,27 +1,22 @@
package org.apache.nutch.parse;
-import org.w3c.dom.DocumentFragment;
-
-// Hadoop imports
import org.apache.hadoop.conf.Configurable;
-
-// Nutch imports
-import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.plugin.TablePluggable;
-import org.apache.nutch.util.hbase.WebTableRow;
+import org.apache.nutch.plugin.FieldPluggable;
+import org.apache.nutch.storage.WebPage;
+import org.w3c.dom.DocumentFragment;
/** Extension point for DOM-based HTML parsers. Permits one to add additional
* metadata to HTML parses. All plugins found which implement this extension
* point are run sequentially on the parse.
*/
-public interface HtmlParseFilter extends TablePluggable, Configurable {
+public interface HtmlParseFilter extends FieldPluggable, Configurable {
/** The name of the extension point. */
final static String X_POINT_ID = HtmlParseFilter.class.getName();
/** Adds metadata or otherwise modifies a parse of HTML content, given
* the DOM tree of a page. */
- Parse filter(String url, WebTableRow row, Parse parseResult,
+ Parse filter(String url, WebPage page, Parse parse,
HTMLMetaTags metaTags, DocumentFragment doc);
-
+
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilters.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilters.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilters.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilters.java Wed Jun 30 10:36:20 2010
@@ -17,17 +17,18 @@
package org.apache.nutch.parse;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.plugin.*;
-import org.apache.nutch.util.ObjectCache;
-import org.apache.nutch.util.hbase.HbaseColumn;
-import org.apache.nutch.util.hbase.WebTableRow;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.ExtensionPoint;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.ObjectCache;
import org.w3c.dom.DocumentFragment;
/** Creates and caches {@link HtmlParseFilter} implementing plugins.*/
@@ -35,49 +36,74 @@ public class HtmlParseFilters {
private HtmlParseFilter[] htmlParseFilters;
+ public static final String HTMLPARSEFILTER_ORDER = "htmlparsefilter.order";
+
public HtmlParseFilters(Configuration conf) {
- ObjectCache objectCache = ObjectCache.get(conf);
- this.htmlParseFilters =
- (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
- if (htmlParseFilters == null) {
- HashMap<String, HtmlParseFilter> filters =
- new HashMap<String, HtmlParseFilter>();
- try {
- ExtensionPoint point =
- PluginRepository.get(conf).getExtensionPoint(HtmlParseFilter.X_POINT_ID);
- if (point == null)
- throw new RuntimeException(HtmlParseFilter.X_POINT_ID + " not found.");
- Extension[] extensions = point.getExtensions();
- for (int i = 0; i < extensions.length; i++) {
- Extension extension = extensions[i];
- HtmlParseFilter parseFilter =
- (HtmlParseFilter) extension.getExtensionInstance();
- if (!filters.containsKey(parseFilter.getClass().getName())) {
- filters.put(parseFilter.getClass().getName(), parseFilter);
- }
- }
- HtmlParseFilter[] htmlParseFilters =
- filters.values().toArray(new HtmlParseFilter[filters.size()]);
- objectCache.setObject(HtmlParseFilter.class.getName(), htmlParseFilters);
- } catch (PluginRuntimeException e) {
- throw new RuntimeException(e);
+ String order = conf.get(HTMLPARSEFILTER_ORDER);
+ ObjectCache objectCache = ObjectCache.get(conf);
+ this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
+ if (htmlParseFilters == null) {
+ /*
+ * If ordered filters are required, prepare array of filters based on
+ * property
+ */
+ String[] orderedFilters = null;
+ if (order != null && !order.trim().equals("")) {
+ orderedFilters = order.split("\\s+");
+ }
+ HashMap<String, HtmlParseFilter> filterMap =
+ new HashMap<String, HtmlParseFilter>();
+ try {
+ ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(HtmlParseFilter.X_POINT_ID);
+ if (point == null)
+ throw new RuntimeException(HtmlParseFilter.X_POINT_ID + " not found.");
+ Extension[] extensions = point.getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ HtmlParseFilter parseFilter = (HtmlParseFilter) extension.getExtensionInstance();
+ if (!filterMap.containsKey(parseFilter.getClass().getName())) {
+ filterMap.put(parseFilter.getClass().getName(), parseFilter);
+ }
+ }
+ HtmlParseFilter[] htmlParseFilters = filterMap.values().toArray(new HtmlParseFilter[filterMap.size()]);
+ /*
+ * If no ordered filters required, just get the filters in an
+ * indeterminate order
+ */
+ if (orderedFilters == null) {
+ objectCache.setObject(HtmlParseFilter.class.getName(), htmlParseFilters);
+ }
+ /* Otherwise run the filters in the required order */
+ else {
+ ArrayList<HtmlParseFilter> filters = new ArrayList<HtmlParseFilter>();
+ for (int i = 0; i < orderedFilters.length; i++) {
+ HtmlParseFilter filter = filterMap
+ .get(orderedFilters[i]);
+ if (filter != null) {
+ filters.add(filter);
}
- this.htmlParseFilters =
- (HtmlParseFilter[])objectCache.getObject(HtmlParseFilter.class.getName());
+ }
+ objectCache.setObject(HtmlParseFilter.class.getName(), filters
+ .toArray(new HtmlParseFilter[filters.size()]));
}
- }
+ } catch (PluginRuntimeException e) {
+ throw new RuntimeException(e);
+ }
+ this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
+ }
+ }
/** Run all defined filters. */
- public Parse filter(String url, WebTableRow row, Parse parse,
- HTMLMetaTags metaTags, DocumentFragment doc) {
+ public Parse filter(String url, WebPage page, Parse parse,
+ HTMLMetaTags metaTags, DocumentFragment doc) {
// loop on each filter
for (HtmlParseFilter htmlParseFilter : htmlParseFilters) {
// call filter interface
- parse = htmlParseFilter.filter(url, row, parse, metaTags, doc);
+ parse = htmlParseFilter.filter(url, page, parse, metaTags, doc);
// any failure on parse obj, return
- if (!parse.getParseStatus().isSuccess()) {
+ if (!ParseStatusUtils.isSuccess(parse.getParseStatus())) {
return parse;
}
}
@@ -85,11 +111,14 @@ public class HtmlParseFilters {
return parse;
}
- public Collection<HbaseColumn> getColumns() {
- Collection<HbaseColumn> columns = new HashSet<HbaseColumn>();
+ public Collection<WebPage.Field> getFields() {
+ Collection<WebPage.Field> fields = new HashSet<WebPage.Field>();
for (HtmlParseFilter htmlParseFilter : htmlParseFilters) {
- columns.addAll(htmlParseFilter.getColumns());
+ Collection<WebPage.Field> pluginFields = htmlParseFilter.getFields();
+ if (pluginFields != null) {
+ fields.addAll(pluginFields);
+ }
}
- return columns;
+ return fields;
}
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parse.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parse.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parse.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parse.java Wed Jun 30 10:36:20 2010
@@ -1,26 +1,18 @@
package org.apache.nutch.parse;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.nutch.parse.Outlink;
-import org.apache.nutch.parse.ParseStatus;
-public class Parse implements Writable {
+public class Parse {
private String text;
private String title;
private Outlink[] outlinks;
- private ParseStatus parseStatus;
+ private org.apache.nutch.storage.ParseStatus parseStatus;
public Parse() {
}
-
+
public Parse(String text, String title, Outlink[] outlinks,
- ParseStatus parseStatus) {
+ org.apache.nutch.storage.ParseStatus parseStatus) {
this.text = text;
this.title = title;
this.outlinks = outlinks;
@@ -39,10 +31,10 @@ public class Parse implements Writable {
return outlinks;
}
- public ParseStatus getParseStatus() {
+ public org.apache.nutch.storage.ParseStatus getParseStatus() {
return parseStatus;
}
-
+
public void setText(String text) {
this.text = text;
}
@@ -55,28 +47,7 @@ public class Parse implements Writable {
this.outlinks = outlinks;
}
- public void setParseStatus(ParseStatus parseStatus) {
+ public void setParseStatus(org.apache.nutch.storage.ParseStatus parseStatus) {
this.parseStatus = parseStatus;
}
-
- public void readFields(DataInput in) throws IOException {
- text = Text.readString(in);
- title = Text.readString(in);
- int numOutlinks = in.readInt();
- outlinks = new Outlink[numOutlinks];
- for (int i = 0; i < numOutlinks; i++) {
- outlinks[i] = Outlink.read(in);
- }
- parseStatus = ParseStatus.read(in);
- }
-
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, text);
- Text.writeString(out, title);
- out.writeInt(outlinks.length);
- for (Outlink outlink : outlinks) {
- outlink.write(out);
- }
- parseStatus.write(out);
- }
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatus.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatus.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatus.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatus.java Wed Jun 30 10:36:20 2010
@@ -1,19 +1,19 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
/*
* Created on Apr 28, 2005
* Author: Andrzej Bialecki <ab@getopt.org>
@@ -28,112 +28,70 @@ import java.io.IOException;
import org.apache.hadoop.io.VersionMismatchException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.conf.Configuration;
/**
* @author Andrzej Bialecki <ab@getopt.org>
*/
-public class ParseStatus implements Writable {
-
- private final static byte VERSION = 2;
-
- // Primary status codes:
-
- /** Parsing was not performed. */
- public static final byte NOTPARSED = 0;
- /** Parsing succeeded. */
- public static final byte SUCCESS = 1;
- /** General failure. There may be a more specific error message in arguments. */
- public static final byte FAILED = 2;
-
- public static final String[] majorCodes = {
- "notparsed",
- "success",
- "failed"
- };
-
- // Secondary success codes go here:
-
- /** Parsed content contains a directive to redirect to another URL.
- * The target URL can be retrieved from the arguments.
- */
- public static final short SUCCESS_REDIRECT = 100;
-
- // Secondary failure codes go here:
-
- /** Parsing failed. An Exception occured (which may be retrieved from the arguments). */
- public static final short FAILED_EXCEPTION = 200;
- /** Parsing failed. Content was truncated, but the parser cannot handle incomplete content. */
- public static final short FAILED_TRUNCATED = 202;
- /** Parsing failed. Invalid format - the content may be corrupted or of wrong type. */
- public static final short FAILED_INVALID_FORMAT = 203;
- /** Parsing failed. Other related parts of the content are needed to complete
- * parsing. The list of URLs to missing parts may be provided in arguments.
- * The Fetcher may decide to fetch these parts at once, then put them into
- * Content.metadata, and supply them for re-parsing.
- */
- public static final short FAILED_MISSING_PARTS = 204;
- /** Parsing failed. There was no content to be parsed - probably caused
- * by errors at protocol stage.
- */
- public static final short FAILED_MISSING_CONTENT = 205;
+@Deprecated
+public class ParseStatus implements Writable, ParseStatusCodes {
+ private final static byte VERSION = 2;
public static final ParseStatus STATUS_NOTPARSED = new ParseStatus(NOTPARSED);
public static final ParseStatus STATUS_SUCCESS = new ParseStatus(SUCCESS);
public static final ParseStatus STATUS_FAILURE = new ParseStatus(FAILED);
-
+
private byte majorCode = 0;
private short minorCode = 0;
private String[] args = null;
-
+
public byte getVersion() {
return VERSION;
}
public ParseStatus() {
-
+
}
-
+
public ParseStatus(int majorCode, int minorCode, String[] args) {
this.args = args;
this.majorCode = (byte)majorCode;
this.minorCode = (short)minorCode;
}
-
+
public ParseStatus(int majorCode) {
this(majorCode, 0, (String[])null);
}
-
+
public ParseStatus(int majorCode, String[] args) {
this(majorCode, 0, args);
}
-
+
public ParseStatus(int majorCode, int minorCode) {
this(majorCode, minorCode, (String[])null);
}
-
+
/** Simplified constructor for passing just a text message. */
public ParseStatus(int majorCode, int minorCode, String message) {
this(majorCode, minorCode, new String[]{message});
}
-
+
/** Simplified constructor for passing just a text message. */
public ParseStatus(int majorCode, String message) {
this(majorCode, 0, new String[]{message});
}
-
+
public ParseStatus(Throwable t) {
this(FAILED, FAILED_EXCEPTION, new String[]{t.toString()});
}
-
+
public static ParseStatus read(DataInput in) throws IOException {
ParseStatus res = new ParseStatus();
res.readFields(in);
return res;
}
-
+
public void readFields(DataInput in) throws IOException {
byte version = in.readByte();
switch(version) {
@@ -150,8 +108,8 @@ public class ParseStatus implements Writ
default:
throw new VersionMismatchException(VERSION, version);
}
- }
-
+ }
+
public void write(DataOutput out) throws IOException {
out.writeByte(VERSION);
out.writeByte(majorCode);
@@ -162,15 +120,15 @@ public class ParseStatus implements Writ
WritableUtils.writeStringArray(out, args);
}
}
-
+
/** A convenience method. Returns true if majorCode is SUCCESS, false
* otherwise.
*/
-
+
public boolean isSuccess() {
return majorCode == SUCCESS;
}
-
+
/** A convenience method. Return a String representation of the first
* argument, or null.
*/
@@ -179,40 +137,34 @@ public class ParseStatus implements Writ
return args[0];
return null;
}
-
+
public String[] getArgs() {
return args;
}
-
+
public int getMajorCode() {
return majorCode;
}
-
+
public int getMinorCode() {
return minorCode;
}
-
+
/** A convenience method. Creates an empty Parse instance,
* which returns this status.
*/
- public Parse getEmptyParse(Configuration conf) {
- return new EmptyParseImpl(this, conf);
- }
-
- /** A convenience method. Creates an empty ParseHbase instance,
- * which returns this status.
- */
- public Parse getEmptyParseHbase(Configuration conf) {
- return new Parse("", "", new Outlink[0], this);
- }
-
+ //public Parse getEmptyParse(Configuration conf) {
+ // return new Parse("", "", new Outlink[0], this);
+ //}
+
/** A convenience method. Creates an empty ParseResult,
* which contains this status.
*/
//public ParseResult getEmptyParseResult(String url, Configuration conf) {
- // return ParseResult.createParseResult(url, getEmptyParse(conf));
+ // return ParseResult.createParseResult(url, getEmptyParse(conf));
//}
-
+
+ @Override
public String toString() {
StringBuffer res = new StringBuffer();
String name = null;
@@ -231,18 +183,18 @@ public class ParseStatus implements Writ
}
return res.toString();
}
-
+
public void setArgs(String[] args) {
this.args = args;
}
-
+
public void setMessage(String msg) {
if (args == null || args.length == 0) {
args = new String[1];
}
args[0] = msg;
}
-
+
public void setMajorCode(byte majorCode) {
this.majorCode = majorCode;
}
@@ -250,14 +202,15 @@ public class ParseStatus implements Writ
public void setMinorCode(short minorCode) {
this.minorCode = minorCode;
}
-
+
+ @Override
public boolean equals(Object o) {
if (o == null) return false;
if (!(o instanceof ParseStatus)) return false;
boolean res = true;
ParseStatus other = (ParseStatus)o;
res = res && (this.majorCode == other.majorCode) &&
- (this.minorCode == other.minorCode);
+ (this.minorCode == other.minorCode);
if (!res) return res;
if (this.args == null) {
if (other.args == null) return true;
@@ -271,12 +224,5 @@ public class ParseStatus implements Writ
}
return true;
}
-
- private static class EmptyParseImpl extends Parse {
-
- public EmptyParseImpl(ParseStatus parseStatus, Configuration conf) {
- super("", "", new Outlink[0], parseStatus);
- }
- }
}
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusCodes.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusCodes.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusCodes.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusCodes.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,44 @@
+package org.apache.nutch.parse;
+
+public interface ParseStatusCodes {
+ // Primary status codes:
+
+ /** Parsing was not performed. */
+ public static final byte NOTPARSED = 0;
+ /** Parsing succeeded. */
+ public static final byte SUCCESS = 1;
+ /** General failure. There may be a more specific error message in arguments. */
+ public static final byte FAILED = 2;
+
+ public static final String[] majorCodes = {
+ "notparsed",
+ "success",
+ "failed"
+ };
+
+ // Secondary success codes go here:
+
+ /** Parsed content contains a directive to redirect to another URL.
+ * The target URL can be retrieved from the arguments.
+ */
+ public static final short SUCCESS_REDIRECT = 100;
+
+ // Secondary failure codes go here:
+
+ /** Parsing failed. An Exception occured (which may be retrieved from the arguments). */
+ public static final short FAILED_EXCEPTION = 200;
+ /** Parsing failed. Content was truncated, but the parser cannot handle incomplete content. */
+ public static final short FAILED_TRUNCATED = 202;
+ /** Parsing failed. Invalid format - the content may be corrupted or of wrong type. */
+ public static final short FAILED_INVALID_FORMAT = 203;
+ /** Parsing failed. Other related parts of the content are needed to complete
+ * parsing. The list of URLs to missing parts may be provided in arguments.
+ * The Fetcher may decide to fetch these parts at once, then put them into
+ * Content.metadata, and supply them for re-parsing.
+ */
+ public static final short FAILED_MISSING_PARTS = 204;
+ /** Parsing failed. There was no content to be parsed - probably caused
+ * by errors at protocol stage.
+ */
+ public static final short FAILED_MISSING_CONTENT = 205;
+}
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusUtils.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusUtils.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusUtils.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusUtils.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,67 @@
+package org.apache.nutch.parse;
+
+import org.apache.avro.generic.GenericArray;
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.storage.ParseStatus;
+import org.apache.nutch.util.TableUtil;
+
+public class ParseStatusUtils {
+
+ public static ParseStatus STATUS_SUCCESS = new ParseStatus();
+
+ static {
+ STATUS_SUCCESS.setMajorCode(ParseStatusCodes.SUCCESS);
+ }
+
+ public static boolean isSuccess(ParseStatus status) {
+ if (status == null) {
+ return false;
+ }
+ return status.getMajorCode() == ParseStatusCodes.SUCCESS;
+ }
+
+ /** A convenience method. Return a String representation of the first
+ * argument, or null.
+ */
+ public static String getMessage(ParseStatus status) {
+ GenericArray<Utf8> args = status.getArgs();
+ if (args != null && args.size() > 0) {
+ return TableUtil.toString(args.iterator().next());
+ }
+ return null;
+ }
+
+ public static String getArg(ParseStatus status, int n) {
+ GenericArray<Utf8> args = status.getArgs();
+ if (args == null) {
+ return null;
+ }
+ int i = 0;
+ for (Utf8 arg : args) {
+ if (i == n) {
+ return TableUtil.toString(arg);
+ }
+ i++;
+ }
+ return null;
+ }
+
+ public static Parse getEmptyParse(Exception e, Configuration conf) {
+ ParseStatus status = new ParseStatus();
+ status.setMajorCode(ParseStatusCodes.FAILED);
+ status.setMinorCode(ParseStatusCodes.FAILED_EXCEPTION);
+ status.addToArgs(new Utf8(e.toString()));
+
+ return new Parse("", "", new Outlink[0], status);
+ }
+
+ public static Parse getEmptyParse(int minorCode, String message, Configuration conf) {
+ ParseStatus status = new ParseStatus();
+ status.setMajorCode(ParseStatusCodes.FAILED);
+ status.setMinorCode(minorCode);
+ status.addToArgs(new Utf8(message));
+
+ return new Parse("", "", new Outlink[0], status);
+ }
+}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java Wed Jun 30 10:36:20 2010
@@ -17,16 +17,28 @@
package org.apache.nutch.parse;
// Commons Logging imports
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+
+import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-// Nutch Imports
-import org.apache.nutch.parse.ParseException;
-import org.apache.nutch.parse.ParserNotFound;
-import org.apache.nutch.util.hbase.WebTableRow;
-
-// Hadoop imports
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.crawl.CrawlStatus;
+import org.apache.nutch.crawl.Signature;
+import org.apache.nutch.crawl.SignatureFactory;
+import org.apache.nutch.crawl.URLWebPage;
+import org.apache.nutch.fetcher.FetcherJob;
+import org.apache.nutch.net.URLFilterException;
+import org.apache.nutch.net.URLFilters;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.TableUtil;
+import org.apache.nutch.util.URLUtil;
/**
* A Utility class containing methods to simply perform parsing utilities such
@@ -37,20 +49,46 @@ import org.apache.hadoop.conf.Configurat
* @author Jérôme Charron
* @author Sébastien Le Callonnec
*/
-public class ParseUtil {
-
+public class ParseUtil extends Configured {
+
/* our log stream */
public static final Log LOG = LogFactory.getLog(ParseUtil.class);
+
+ private Configuration conf;
+
+ private Signature sig;
+ private URLFilters filters;
+ private URLNormalizers normalizers;
+ private int maxOutlinks;
+ private boolean ignoreExternalLinks;
private ParserFactory parserFactory;
-
+
/**
- *
+ *
* @param conf
*/
public ParseUtil(Configuration conf) {
- this.parserFactory = new ParserFactory(conf);
+ super(conf);
+ setConf(conf);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
}
-
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ parserFactory = new ParserFactory(conf);
+ sig = SignatureFactory.getSignature(conf);
+ filters = new URLFilters(conf);
+ normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_OUTLINK);
+ int maxOutlinksPerPage = conf.getInt("db.max.outlinks.per.page", 100);
+ maxOutlinks = (maxOutlinksPerPage < 0) ? Integer.MAX_VALUE : maxOutlinksPerPage;
+ ignoreExternalLinks = conf.getBoolean("db.ignore.external.links", false);
+ }
+
/**
* Performs a parse by iterating through a List of preferred {@link Parser}s
* until a successful parse is performed and a {@link Parse} object is
@@ -59,35 +97,142 @@ public class ParseUtil {
*
* @throws ParseException If no suitable parser is found to perform the parse.
*/
- public Parse parse(String url, WebTableRow row) throws ParseException {
+ public Parse parse(String url, WebPage page) throws ParseException {
Parser[] parsers = null;
-
- String contentType = row.getContentType();
-
+
+ String contentType = TableUtil.toString(page.getContentType());
+
try {
parsers = this.parserFactory.getParsers(contentType, url);
} catch (ParserNotFound e) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("No suitable parser found when trying to parse content " + url +
- " of type " + contentType);
- }
+ LOG.warn("No suitable parser found when trying to parse content " + url +
+ " of type " + contentType);
throw new ParseException(e.getMessage());
}
-
- Parse parse = null;
+
for (int i=0; i<parsers.length; i++) {
if (LOG.isDebugEnabled()) {
LOG.debug("Parsing [" + url + "] with [" + parsers[i] + "]");
}
- parse = parsers[i].getParse(url, row);
- if (parse != null)
+ Parse parse = parsers[i].getParse(url, page);
+ if (ParseStatusUtils.isSuccess(parse.getParseStatus())) {
return parse;
+ }
}
-
- if (LOG.isWarnEnabled()) {
- LOG.warn("Unable to successfully parse content " + url +
- " of type " + contentType);
- }
+
+ LOG.warn("Unable to successfully parse content " + url +
+ " of type " + contentType);
return null;
}
+
+ /**
+ * Parses given web page and stores parsed content within page. Returns
+ * a pair of <String, WebPage> if a meta-redirect is discovered
+ * @param key
+ * @param page
+ * @return newly-discovered webpage (via a meta-redirect)
+ */
+ public URLWebPage process(String key, WebPage page) {
+ URLWebPage redirectedPage = null;
+ String url = TableUtil.unreverseUrl(key);
+ byte status = (byte) page.getStatus();
+ if (status != CrawlStatus.STATUS_FETCHED) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping " + url + " as status is: " + CrawlStatus.getName(status));
+ }
+ return redirectedPage;
+ }
+
+ Parse parse;
+ try {
+ parse = parse(url, page);
+ } catch (final Exception e) {
+ LOG.warn("Error parsing: " + url + ": " + StringUtils.stringifyException(e));
+ return redirectedPage;
+ }
+
+ if (parse == null) {
+ return redirectedPage;
+ }
+
+ final byte[] signature = sig.calculate(page);
+
+ org.apache.nutch.storage.ParseStatus pstatus = parse.getParseStatus();
+ page.setParseStatus(pstatus);
+ if (ParseStatusUtils.isSuccess(pstatus)) {
+ if (pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) {
+ String newUrl = ParseStatusUtils.getMessage(pstatus);
+ int refreshTime = Integer.parseInt(ParseStatusUtils.getArg(pstatus, 1));
+ try {
+ newUrl = normalizers.normalize(newUrl, URLNormalizers.SCOPE_FETCHER);
+ newUrl = filters.filter(newUrl);
+ } catch (URLFilterException e) {
+ return redirectedPage; // TODO: is this correct
+ } catch (MalformedURLException e) {
+ return redirectedPage;
+ }
+ if (newUrl == null || newUrl.equals(url)) {
+ String reprUrl = URLUtil.chooseRepr(url, newUrl,
+ refreshTime < FetcherJob.PERM_REFRESH_TIME);
+ WebPage newWebPage = new WebPage();
+ page.setReprUrl(new Utf8(reprUrl));
+ page.putToMetadata(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL);
+ redirectedPage = new URLWebPage(reprUrl, newWebPage);
+ }
+ } else {
+ page.setText(new Utf8(parse.getText()));
+ page.setTitle(new Utf8(parse.getTitle()));
+ ByteBuffer prevSig = page.getSignature();
+ if (prevSig != null) {
+ page.setPrevSignature(prevSig);
+ }
+ page.setSignature(ByteBuffer.wrap(signature));
+ if (page.getOutlinks() != null) {
+ page.getOutlinks().clear();
+ }
+ final Outlink[] outlinks = parse.getOutlinks();
+ final int count = 0;
+ String fromHost;
+ if (ignoreExternalLinks) {
+ try {
+ fromHost = new URL(url).getHost().toLowerCase();
+ } catch (final MalformedURLException e) {
+ fromHost = null;
+ }
+ } else {
+ fromHost = null;
+ }
+ for (int i = 0; count < maxOutlinks && i < outlinks.length; i++) {
+ String toUrl = outlinks[i].getToUrl();
+ try {
+ toUrl = normalizers.normalize(toUrl, URLNormalizers.SCOPE_OUTLINK);
+ toUrl = filters.filter(toUrl);
+ } catch (final URLFilterException e) {
+ continue;
+ }
+ catch (MalformedURLException e2){
+ continue;
+ }
+ if (toUrl == null) {
+ continue;
+ }
+ String toHost;
+ if (ignoreExternalLinks) {
+ try {
+ toHost = new URL(toUrl).getHost().toLowerCase();
+ } catch (final MalformedURLException e) {
+ toHost = null;
+ }
+ if (toHost == null || !toHost.equals(fromHost)) { // external links
+ continue; // skip it
+ }
+ }
+
+ page.putToOutlinks(new Utf8(toUrl), new Utf8(outlinks[i].getAnchor()));
+ }
+ Mark.PARSE_MARK.putMark(page, Mark.FETCH_MARK.checkMark(page));
+ }
+ }
+ return redirectedPage;
+ }
}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parser.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parser.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parser.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parser.java Wed Jun 30 10:36:20 2010
@@ -19,29 +19,24 @@ package org.apache.nutch.parse;
// Hadoop imports
import org.apache.hadoop.conf.Configurable;
-
-// Nutch imports
-import org.apache.nutch.plugin.TablePluggable;
-import org.apache.nutch.util.hbase.WebTableRow;
+import org.apache.nutch.plugin.FieldPluggable;
+import org.apache.nutch.storage.WebPage;
/** A parser for content generated by a {@link org.apache.nutch.protocol.Protocol}
* implementation. This interface is implemented by extensions. Nutch's core
* contains no page parsing code.
*/
-public interface Parser extends TablePluggable, Configurable {
+public interface Parser extends FieldPluggable, Configurable {
/** The name of the extension point. */
public final static String X_POINT_ID = Parser.class.getName();
- /**
+ /**
* <p>
- * This method takes a Hbase table row and returns a {@link Parse} instance
+ * This method parses content in WebPage instance
* </p>
- *
+ *
* @param url Page's URL
- * @param row url's row. Parse plugins can add their updates here. They
- * will be reflected in hbase. Plugins must not update for items already
- * in ParseHbase
- * @return Parse
+ * @param page
*/
- Parse getParse(String url, WebTableRow row);
+ Parse getParse(String url, WebPage page);
}
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserChecker.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserChecker.java (added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserChecker.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.parse;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.ProtocolFactory;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.NutchConfiguration;
+
+/**
+ * Parser checker, useful for testing parser.
+ *
+ * @author John Xing
+ */
+
+public class ParserChecker {
+
+ public static final Log LOG = LogFactory.getLog(ParserChecker.class);
+
+ public ParserChecker() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ boolean dumpText = false;
+ boolean force = false;
+ String contentType = null;
+ String url = null;
+
+ String usage = "Usage: ParserChecker [-dumpText] [-forceAs mimeType] url";
+
+ if (args.length == 0) {
+ System.err.println(usage);
+ System.exit(-1);
+ }
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-forceAs")) {
+ force = true;
+ contentType = args[++i];
+ } else if (args[i].equals("-dumpText")) {
+ dumpText = true;
+ } else if (i != args.length - 1) {
+ System.err.println(usage);
+ System.exit(-1);
+ } else {
+ url = args[i];
+ }
+ }
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("fetching: " + url);
+ }
+
+ Configuration conf = NutchConfiguration.create();
+ ProtocolFactory factory = new ProtocolFactory(conf);
+ Protocol protocol = factory.getProtocol(url);
+ WebPage page = new WebPage();
+ Content content = protocol.getProtocolOutput(url, page).getContent();
+ page.setBaseUrl(new org.apache.avro.util.Utf8(url));
+ page.setContent(ByteBuffer.wrap(content.getContent()));
+
+ if (force) {
+ content.setContentType(contentType);
+ } else {
+ contentType = content.getContentType();
+ }
+
+ if (contentType == null) {
+ System.err.println("");
+ System.exit(-1);
+ }
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("parsing: " + url);
+ LOG.info("contentType: " + contentType);
+ }
+
+ page.setContentType(new Utf8(contentType));
+
+ Parse parse = new ParseUtil(conf).parse(url, page);
+
+ if (parse==null){
+ System.err.println("Problem with parse - check log");
+ System.exit(-1);
+ }
+
+ System.out.print("---------\nUrl\n---------------\n");
+ System.out.print(url+"\n");
+ System.out.print("---------\nMetadata\n---------\n");
+ Map<Utf8, ByteBuffer> metadata = page.getMetadata();
+ StringBuffer sb = new StringBuffer();
+ if (metadata != null) {
+ Iterator<Entry<Utf8, ByteBuffer>> iterator = metadata.entrySet()
+ .iterator();
+ while (iterator.hasNext()) {
+ Entry<Utf8, ByteBuffer> entry = iterator.next();
+ sb.append(entry.getKey().toString()).append(" : \t")
+ .append(Bytes.toString(entry.getValue().array())).append("\n");
+ }
+ System.out.print(sb.toString());
+ }
+ if (dumpText) {
+ System.out.print("---------\nParseText\n---------\n");
+ System.out.print(parse.getText());
+ }
+
+ System.exit(0);
+ }
+}
Modified: nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserFactory.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserFactory.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserFactory.java (original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserFactory.java Wed Jun 30 10:36:20 2010
@@ -23,41 +23,33 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-// Commons Logging imports
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-
-// Nutch imports
-import org.apache.nutch.parse.ParsePluginList;
-import org.apache.nutch.parse.ParsePluginsReader;
-import org.apache.nutch.parse.ParserNotFound;
import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.ExtensionPoint;
-import org.apache.nutch.plugin.PluginRuntimeException;
import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.ObjectCache;
-import org.apache.nutch.util.hbase.HbaseColumn;
/** Creates and caches {@link Parser} plugins.*/
public final class ParserFactory {
-
+
public static final Log LOG = LogFactory.getLog(ParserFactory.class);
-
+
/** Wildcard for default plugins. */
public static final String DEFAULT_PLUGIN = "*";
-
+
/** Empty extension list for caching purposes. */
private final List<Extension> EMPTY_EXTENSION_LIST =
new ArrayList<Extension>();
-
- private Configuration conf;
- private ExtensionPoint extensionPoint;
+
+ private final Configuration conf;
+ private final ExtensionPoint extensionPoint;
private ParsePluginList parsePluginList;
public ParserFactory(Configuration conf) {
@@ -78,9 +70,9 @@ public final class ParserFactory {
throw new RuntimeException(
"Parse Plugins preferences could not be loaded.");
}
- }
-
-
+ }
+
+
/**
* Function returns an array of {@link Parser}s for a given content type.
*
@@ -107,12 +99,12 @@ public final class ParserFactory {
*/
public Parser[] getParsers(String contentType, String url)
throws ParserNotFound {
-
+
List<Parser> parsers = null;
List<Extension> parserExts = null;
-
+
ObjectCache objectCache = ObjectCache.get(conf);
-
+
// TODO once the MimeTypes is available
// parsers = getExtensions(MimeUtils.map(contentType));
// if (parsers != null) {
@@ -151,7 +143,7 @@ public final class ParserFactory {
}
return parsers.toArray(new Parser[]{});
}
-
+
/**
* Function returns a {@link Parser} instance with the specified
* <code>extId</code>, representing its extension ID. If the Parser
@@ -161,7 +153,7 @@ public final class ParserFactory {
* will return the already instantiated Parser. Otherwise, if it has to
* instantiate the Parser itself , then this function will cache that Parser
* in the internal <code>PARSER_CACHE</code>.
- *
+ *
* @param id The string extension ID (e.g.,
* "org.apache.nutch.parse.rss.RSSParser",
* "org.apache.nutch.parse.rtf.RTFParseFactory") of the {@link Parser}
@@ -178,7 +170,7 @@ public final class ParserFactory {
Extension parserExt = null;
ObjectCache objectCache = ObjectCache.get(conf);
-
+
if (id != null) {
parserExt = getExtension(extensions, id);
}
@@ -189,12 +181,12 @@ public final class ParserFactory {
if (parserExt == null) {
throw new ParserNotFound("No Parser Found for id [" + id + "]");
}
-
- // first check the cache
+
+ // first check the cache
if (objectCache.getObject(parserExt.getId()) != null) {
return (Parser) objectCache.getObject(parserExt.getId());
- // if not found in cache, instantiate the Parser
+ // if not found in cache, instantiate the Parser
} else {
try {
Parser p = (Parser) parserExt.getExtensionInstance();
@@ -210,31 +202,34 @@ public final class ParserFactory {
}
}
}
-
- public Collection<HbaseColumn> getColumns() {
- Set<HbaseColumn> columns = new HashSet<HbaseColumn>();
+
+ public Collection<WebPage.Field> getFields() {
+ Set<WebPage.Field> columns = new HashSet<WebPage.Field>();
Extension[] extensions = this.extensionPoint.getExtensions();
for (Extension ext : extensions) {
try {
Parser parser = (Parser) ext.getExtensionInstance();
- columns.addAll(parser.getColumns());
+ Collection<WebPage.Field> pluginFields = parser.getFields();
+ if (pluginFields != null) {
+ columns.addAll(pluginFields);
+ }
} catch (PluginRuntimeException e) {
- // ignore
+ LOG.error(e);
}
}
return columns;
}
-
+
/**
* Finds the best-suited parse plugin for a given contentType.
- *
+ *
* @param contentType Content-Type for which we seek a parse plugin.
* @return a list of extensions to be used for this contentType.
* If none, returns <code>null</code>.
*/
@SuppressWarnings("unchecked")
protected List<Extension> getExtensions(String contentType) {
-
+
ObjectCache objectCache = ObjectCache.get(conf);
// First of all, tries to clean the content-type
String type = null;
@@ -247,7 +242,7 @@ public final class ParserFactory {
if (extensions == EMPTY_EXTENSION_LIST) {
return null;
}
-
+
if (extensions == null) {
extensions = findExtensions(type);
if (extensions != null) {
@@ -260,20 +255,20 @@ public final class ParserFactory {
}
return extensions;
}
-
+
/**
* searches a list of suitable parse plugins for the given contentType.
* <p>It first looks for a preferred plugin defined in the parse-plugin
* file. If none is found, it returns a list of default plugins.
- *
+ *
* @param contentType Content-Type for which we seek a parse plugin.
* @return List - List of extensions to be used for this contentType.
* If none, returns null.
*/
private List<Extension> findExtensions(String contentType) {
-
+
Extension[] extensions = this.extensionPoint.getExtensions();
-
+
// Look for a preferred plugin.
List<String> parsePluginList =
this.parsePluginList.getPluginList(contentType);
@@ -282,12 +277,12 @@ public final class ParserFactory {
if (extensionList != null) {
return extensionList;
}
-
+
// If none found, look for a default plugin.
parsePluginList = this.parsePluginList.getPluginList(DEFAULT_PLUGIN);
return matchExtensions(parsePluginList, extensions, DEFAULT_PLUGIN);
}
-
+
/**
* Tries to find a suitable parser for the given contentType.
* <ol>
@@ -305,27 +300,27 @@ public final class ParserFactory {
private List<Extension> matchExtensions(List<String> plugins,
Extension[] extensions,
String contentType) {
-
+
List<Extension> extList = new ArrayList<Extension>();
if (plugins != null) {
-
+
for (String parsePluginId : plugins) {
-
+
Extension ext = getExtension(extensions, parsePluginId, contentType);
// the extension returned may be null
// that means that it was not enabled in the plugin.includes
// nutch conf property, but it was mapped in the
// parse-plugins.xml
- // file.
+ // file.
// OR it was enabled in plugin.includes, but the plugin's plugin.xml
// file does not claim that the plugin supports the specified mimeType
// in either case, LOG the appropriate error message to WARN level
-
+
if (ext == null) {
//try to get it just by its pluginId
ext = getExtension(extensions, parsePluginId);
-
- if (LOG.isWarnEnabled()) {
+
+ if (LOG.isWarnEnabled()) {
if (ext != null) {
// plugin was enabled via plugin.includes
// its plugin.xml just doesn't claim to support that
@@ -337,10 +332,10 @@ public final class ParserFactory {
contentType);
} else {
// plugin wasn't enabled via plugin.includes
- LOG.warn("ParserFactory: Plugin: " + parsePluginId +
+ LOG.warn("ParserFactory: Plugin: " + parsePluginId +
" mapped to contentType " + contentType +
" via parse-plugins.xml, but not enabled via " +
- "plugin.includes in nutch-default.xml");
+ "plugin.includes in nutch-default.xml");
}
}
}
@@ -350,7 +345,7 @@ public final class ParserFactory {
extList.add(ext);
}
}
-
+
} else {
// okay, there were no list of plugins defined for
// this mimeType, however, there may be plugins registered
@@ -359,18 +354,29 @@ public final class ParserFactory {
// so, iterate through the list of extensions and if you find
// any extensions where this is the case, throw a
// NotMappedParserException
-
+
for (int i=0; i<extensions.length; i++) {
if (extensions[i].getAttribute("contentType") != null
&& extensions[i].getAttribute("contentType").equals(
contentType)) {
extList.add(extensions[i]);
}
+ else if ("*".equals(extensions[i].getAttribute("contentType"))){
+ extList.add(0, extensions[i]);
+ }
}
-
+
if (extList.size() > 0) {
if (LOG.isInfoEnabled()) {
- LOG.info("The parsing plugins: " + extList +
+ StringBuffer extensionsIDs = new StringBuffer("[");
+ boolean isFirst = true;
+ for (Extension ext : extList){
+ if (!isFirst) extensionsIDs.append(" - ");
+ else isFirst=false;
+ extensionsIDs.append(ext.getId());
+ }
+ extensionsIDs.append("]");
+ LOG.info("The parsing plugins: " + extensionsIDs.toString() +
" are enabled via the plugin.includes system " +
"property, and all claim to support the content type " +
contentType + ", but they are not mapped to it in the " +
@@ -381,16 +387,16 @@ public final class ParserFactory {
"contentType " + contentType);
}
}
-
+
return (extList.size() > 0) ? extList : null;
}
private boolean match(Extension extension, String id, String type) {
return ((id.equals(extension.getId())) &&
- (type.equals(extension.getAttribute("contentType")) ||
+ (type.equals(extension.getAttribute("contentType")) || extension.getAttribute("contentType").equals("*") ||
type.equals(DEFAULT_PLUGIN)));
}
-
+
/** Get an extension from its id and supported content-type. */
private Extension getExtension(Extension[] list, String id, String type) {
for (int i=0; i<list.length; i++) {
@@ -400,7 +406,7 @@ public final class ParserFactory {
}
return null;
}
-
+
private Extension getExtension(Extension[] list, String id) {
for (int i=0; i<list.length; i++) {
if (id.equals(list[i].getId())) {
@@ -409,7 +415,7 @@ public final class ParserFactory {
}
return null;
}
-
+
private Extension getExtensionFromAlias(Extension[] list, String id) {
return getExtension(list, parsePluginList.getAliases().get(id));
}