You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2013/03/07 12:19:25 UTC
svn commit: r1453776 - in /nutch/trunk: ./ conf/ ivy/ src/bin/
src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/indexer/
src/java/org/apache/nutch/indexer/solr/ src/plugin/
src/plugin/indexer-solr/ src/plugin/indexer-solr/src/ src/plugin/inde...
Author: jnioche
Date: Thu Mar 7 11:19:24 2013
New Revision: 1453776
URL: http://svn.apache.org/r1453776
Log:
NUTCH-1047 Pluggable indexing backends
Added:
nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java
nutch/trunk/src/plugin/indexer-solr/
nutch/trunk/src/plugin/indexer-solr/build.xml
nutch/trunk/src/plugin/indexer-solr/ivy.xml
nutch/trunk/src/plugin/indexer-solr/plugin.xml
nutch/trunk/src/plugin/indexer-solr/src/
nutch/trunk/src/plugin/indexer-solr/src/java/
nutch/trunk/src/plugin/indexer-solr/src/java/org/
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java
nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
Removed:
nutch/trunk/src/java/org/apache/nutch/indexer/NutchIndexWriter.java
nutch/trunk/src/java/org/apache/nutch/indexer/NutchIndexWriterFactory.java
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrClean.java
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/build.xml
nutch/trunk/conf/nutch-default.xml
nutch/trunk/default.properties
nutch/trunk/ivy/ivy.xml
nutch/trunk/src/bin/nutch
nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java
nutch/trunk/src/plugin/build.xml
nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Mar 7 11:19:24 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1047 Pluggable indexing backends (jnioche)
+
* NUTCH-1536 Ant build file has hardcoded conf dir location (zm via lewismc)
* NUTCH-1420 Get rid of the dreaded � (markus via lewismc)
Modified: nutch/trunk/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/build.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/build.xml (original)
+++ nutch/trunk/build.xml Thu Mar 7 11:19:24 2013
@@ -133,7 +133,7 @@
<!-- build the main artifact -->
<jar jarfile="${maven-jar}" basedir="${build.classes}" />
-
+
<!-- build the javadoc artifact -->
<javadoc
destdir="${release.dir}/javadoc"
@@ -157,6 +157,7 @@
<packageset dir="${plugins.dir}/index-metadata/src/java"/>
<packageset dir="${plugins.dir}/index-more/src/java"/>
<packageset dir="${plugins.dir}/index-static/src/java"/>
+ <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
<packageset dir="${plugins.dir}/language-identifier/src/java"/>
<packageset dir="${plugins.dir}/lib-http/src/java"/>
<packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -208,6 +209,7 @@
<group title="Scoring Plugins" packages="${plugins.scoring}"/>
<group title="Parse Plugins" packages="${plugins.parse}"/>
<group title="Indexing Filter Plugins" packages="${plugins.index}"/>
+ <group title="Indexer Plugins" packages="${plugins.indexer}"/>
<group title="Misc. Plugins" packages="${plugins.misc}"/>
</javadoc>
<jar jarfile="${maven-javadoc-jar}">
@@ -545,6 +547,7 @@
<packageset dir="${plugins.dir}/index-metadata/src/java"/>
<packageset dir="${plugins.dir}/index-more/src/java"/>
<packageset dir="${plugins.dir}/index-static/src/java"/>
+ <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
<packageset dir="${plugins.dir}/language-identifier/src/java"/>
<packageset dir="${plugins.dir}/lib-http/src/java"/>
<packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -594,6 +597,7 @@
<group title="Scoring Plugins" packages="${plugins.scoring}"/>
<group title="Parse Plugins" packages="${plugins.parse}"/>
<group title="Indexing Filter Plugins" packages="${plugins.index}"/>
+ <group title="Indexer Plugins" packages="${plugins.indexer}"/>
<group title="Misc. Plugins" packages="${plugins.misc}"/>
</javadoc>
<!-- Copy the plugin.dtd file to the plugin doc-files dir -->
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Thu Mar 7 11:19:24 2013
@@ -990,7 +990,7 @@
<property>
<name>plugin.includes</name>
- <value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|scoring-opic|urlnormalizer-(pass|regex|basic)</value>
+ <value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)</value>
<description>Regular expression naming plugin directory names to
include. Any plugin not matching this expression is excluded.
In any case you need at least include the nutch-extensionpoints plugin. By
Modified: nutch/trunk/default.properties
URL: http://svn.apache.org/viewvc/nutch/trunk/default.properties?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/default.properties (original)
+++ nutch/trunk/default.properties Thu Mar 7 11:19:24 2013
@@ -150,6 +150,12 @@ plugins.index=\
org.apache.nutch.indexer.urlmeta*
#
+# Indexing Backend Plugins
+#
+plugins.indexer=\
+ org.apache.nutch.indexwriter.solr*
+
+#
# Misc. Plugins
#
# (gathers plugins that cannot be dispatched
Modified: nutch/trunk/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Thu Mar 7 11:19:24 2013
@@ -13,10 +13,14 @@
<ivy-module version="1.0">
<info organisation="org.apache.nutch" module="nutch">
- <license name="Apache 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt/"/>
+ <license name="Apache 2.0"
+ url="http://www.apache.org/licenses/LICENSE-2.0.txt/" />
<ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org" />
- <description homepage="http://nutch.apache.org">Nutch is an open source web-search software. It builds on
- Hadoop, Tika and Solr, adding web-specifics, such as a crawler, a link-graph database etc.
+ <description homepage="http://nutch.apache.org">Nutch is an open source web-search
+ software. It builds on
+ Hadoop, Tika and Solr, adding web-specifics,
+ such as a crawler, a link-graph
+ database etc.
</description>
</info>
@@ -31,8 +35,12 @@
<dependencies>
<dependency org="org.apache.solr" name="solr-solrj" rev="3.4.0"
- conf="*->default"/>
- <dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.1" conf="*->master" />
+ conf="*->default" />
+
+ <dependency org="org.slf4j" name="slf4j-api" rev="1.6.1"
+ conf="*->master" />
+ <dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.1"
+ conf="*->master" />
<dependency org="commons-lang" name="commons-lang" rev="2.6"
conf="*->default" />
@@ -55,15 +63,16 @@
<dependency org="org.apache.tika" name="tika-core" rev="1.2" />
<dependency org="com.ibm.icu" name="icu4j" rev="4.0.1" />
-
- <dependency org="org.mortbay.jetty" name="jetty-client" rev="6.1.22" />
+
+ <dependency org="org.mortbay.jetty" name="jetty-client"
+ rev="6.1.22" />
<dependency org="log4j" name="log4j" rev="1.2.15" conf="*->master" />
<dependency org="xerces" name="xercesImpl" rev="2.9.1" />
<dependency org="xerces" name="xmlParserAPIs" rev="2.6.2" />
<dependency org="oro" name="oro" rev="2.0.8" />
-
+
<dependency org="com.google.guava" name="guava" rev="11.0.2" />
<!--Configuration: test -->
@@ -78,10 +87,10 @@
<dependency org="org.mortbay.jetty" name="jetty-util" rev="6.1.22"
conf="test->default" />
- <!--global exclusion-->
- <exclude module="jmxtools" />
- <exclude module="jms" />
- <exclude module="jmxri" />
+ <!--global exclusion -->
+ <exclude module="jmxtools" />
+ <exclude module="jms" />
+ <exclude module="jmxri" />
</dependencies>
Modified: nutch/trunk/src/bin/nutch
URL: http://svn.apache.org/viewvc/nutch/trunk/src/bin/nutch?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/bin/nutch (original)
+++ nutch/trunk/src/bin/nutch Thu Mar 7 11:19:24 2013
@@ -61,9 +61,11 @@ if [ $# = 0 ]; then
echo " updatedb update crawl db from segments after fetching"
echo " invertlinks create a linkdb from parsed segments"
echo " mergelinkdb merge linkdb-s, with optional filtering"
+ echo " index run the plugin-based indexer on parsed segments and linkdb"
echo " solrindex run the solr indexer on parsed segments and linkdb"
echo " solrdedup remove duplicates from solr"
echo " solrclean remove HTTP 301 and 404 documents from solr"
+ echo " clean remove HTTP 301 and 404 documents from indexing backends configured via plugins"
echo " parsechecker check the parser for a given url"
echo " indexchecker check the indexing filters for a given url"
echo " domainstats calculate domain statistics from crawldb"
@@ -219,11 +221,17 @@ elif [ "$COMMAND" = "invertlinks" ] ; th
elif [ "$COMMAND" = "mergelinkdb" ] ; then
CLASS=org.apache.nutch.crawl.LinkDbMerger
elif [ "$COMMAND" = "solrindex" ] ; then
- CLASS=org.apache.nutch.indexer.solr.SolrIndexer
+ CLASS="org.apache.nutch.indexer.IndexingJob -D solr.server.url=$1"
+ shift
+elif [ "$COMMAND" = "index" ] ; then
+ CLASS=org.apache.nutch.indexer.IndexingJob
elif [ "$COMMAND" = "solrdedup" ] ; then
CLASS=org.apache.nutch.indexer.solr.SolrDeleteDuplicates
elif [ "$COMMAND" = "solrclean" ] ; then
- CLASS=org.apache.nutch.indexer.solr.SolrClean
+ CLASS="org.apache.nutch.indexer.CleaningJob -D solr.server.url=$2 $1"
+ shift; shift
+elif [ "$COMMAND" = "clean" ] ; then
+ CLASS=org.apache.nutch.indexer.CleaningJob
elif [ "$COMMAND" = "parsechecker" ] ; then
CLASS=org.apache.nutch.parse.ParserChecker
elif [ "$COMMAND" = "indexchecker" ] ; then
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java Thu Mar 7 11:19:24 2013
@@ -31,8 +31,8 @@ import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.parse.ParseSegment;
+import org.apache.nutch.indexer.IndexingJob;
import org.apache.nutch.indexer.solr.SolrDeleteDuplicates;
-import org.apache.nutch.indexer.solr.SolrIndexer;
import org.apache.nutch.util.HadoopFSUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -96,6 +96,11 @@ public class Crawl extends Configured im
if (solrUrl == null) {
LOG.warn("solrUrl is not set, indexing will be skipped...");
}
+ else {
+ // for simplicity assume that SOLR is used
+ // and pass its URL via conf
+ getConf().set("solr.server.url", solrUrl);
+ }
FileSystem fs = FileSystem.get(job);
@@ -145,9 +150,11 @@ public class Crawl extends Configured im
if (solrUrl != null) {
// index, dedup & merge
FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
- SolrIndexer indexer = new SolrIndexer(getConf());
- indexer.indexSolr(solrUrl, crawlDb, linkDb,
- Arrays.asList(HadoopFSUtil.getPaths(fstats)));
+
+ IndexingJob indexer = new IndexingJob(getConf());
+ indexer.index(crawlDb, linkDb,
+ Arrays.asList(HadoopFSUtil.getPaths(fstats)));
+
SolrDeleteDuplicates dedup = new SolrDeleteDuplicates();
dedup.setConf(getConf());
dedup.dedup(solrUrl);
Added: nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.CrawlDb;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.TimingUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The class scans CrawlDB looking for entries with status DB_GONE (404) and
+ * sends delete requests to indexers for those documents.
+ *
+ */
+
+public class CleaningJob implements Tool {
+ public static final Logger LOG = LoggerFactory.getLogger(CleaningJob.class);
+ private Configuration conf;
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static class DBFilter implements
+ Mapper<Text, CrawlDatum, ByteWritable, Text> {
+ private ByteWritable OUT = new ByteWritable(CrawlDatum.STATUS_DB_GONE);
+
+ @Override
+ public void configure(JobConf arg0) {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void map(Text key, CrawlDatum value,
+ OutputCollector<ByteWritable, Text> output, Reporter reporter)
+ throws IOException {
+
+ if (value.getStatus() == CrawlDatum.STATUS_DB_GONE) {
+ output.collect(OUT, key);
+ }
+ }
+ }
+
+ public static class DeleterReducer implements
+ Reducer<ByteWritable, Text, Text, ByteWritable> {
+ private static final int NUM_MAX_DELETE_REQUEST = 1000;
+ private int numDeletes = 0;
+ private int totalDeleted = 0;
+
+ private boolean noCommit = false;
+
+ IndexWriters writers = null;
+
+ @Override
+ public void configure(JobConf job) {
+ writers = new IndexWriters(job);
+ try {
+ writers.open(job, "Deletion");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ noCommit = job.getBoolean("noCommit", false);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // BUFFERING OF CALLS TO INDEXER SHOULD BE HANDLED AT INDEXER LEVEL
+ // if (numDeletes > 0) {
+ // LOG.info("CleaningJob: deleting " + numDeletes + " documents");
+ // // TODO updateRequest.process(solr);
+ // totalDeleted += numDeletes;
+ // }
+
+ writers.close();
+
+ if (totalDeleted > 0 && !noCommit) {
+ writers.commit();
+ }
+
+ LOG.info("CleaningJob: deleted a total of " + totalDeleted
+ + " documents");
+ }
+
+ @Override
+ public void reduce(ByteWritable key, Iterator<Text> values,
+ OutputCollector<Text, ByteWritable> output, Reporter reporter)
+ throws IOException {
+ while (values.hasNext()) {
+ Text document = values.next();
+ writers.delete(document.toString());
+ totalDeleted++;
+ reporter.incrCounter("CleaningJobStatus", "Deleted documents",
+ 1);
+ // if (numDeletes >= NUM_MAX_DELETE_REQUEST) {
+ // LOG.info("CleaningJob: deleting " + numDeletes
+ // + " documents");
+ // // TODO updateRequest.process(solr);
+ // // TODO updateRequest = new UpdateRequest();
+ // writers.delete(key.toString());
+ // totalDeleted += numDeletes;
+ // numDeletes = 0;
+ // }
+ }
+ }
+ }
+
+ public void delete(String crawldb, boolean noCommit) throws IOException {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ long start = System.currentTimeMillis();
+ LOG.info("CleaningJob: starting at " + sdf.format(start));
+
+ JobConf job = new NutchJob(getConf());
+
+ FileInputFormat.addInputPath(job, new Path(crawldb,
+ CrawlDb.CURRENT_NAME));
+ job.setBoolean("noCommit", noCommit);
+ job.setInputFormat(SequenceFileInputFormat.class);
+ job.setOutputFormat(NullOutputFormat.class);
+ job.setMapOutputKeyClass(ByteWritable.class);
+ job.setMapOutputValueClass(Text.class);
+ job.setMapperClass(DBFilter.class);
+ job.setReducerClass(DeleterReducer.class);
+
+ // need to expicitely allow deletions
+ job.setBoolean(IndexerMapReduce.INDEXER_DELETE, true);
+
+ JobClient.runJob(job);
+
+ long end = System.currentTimeMillis();
+ LOG.info("CleaningJob: finished at " + sdf.format(end) + ", elapsed: "
+ + TimingUtil.elapsedTime(start, end));
+ }
+
+ public int run(String[] args) throws IOException {
+ if (args.length < 1) {
+ System.err.println("Usage: CleaningJob <crawldb> [-noCommit]");
+ IndexWriters writers = new IndexWriters(getConf());
+ System.err.println(writers.describe());
+ return 1;
+ }
+
+ boolean noCommit = false;
+ if (args.length == 2 && args[1].equals("-noCommit")) {
+ noCommit = true;
+ }
+
+ try {
+ delete(args[0], noCommit);
+ } catch (final Exception e) {
+ LOG.error("CleaningJob: " + StringUtils.stringifyException(e));
+ System.err.println("ERROR CleaningJob: "
+ + StringUtils.stringifyException(e));
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int result = ToolRunner.run(NutchConfiguration.create(),
+ new CleaningJob(), args);
+ System.exit(result);
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.plugin.Pluggable;
+
+public interface IndexWriter extends Pluggable, Configurable {
+ /** The name of the extension point. */
+ final static String X_POINT_ID = IndexWriter.class.getName();
+
+ public void open(JobConf job, String name) throws IOException;
+
+ public void write(NutchDocument doc) throws IOException;
+
+ public void delete(String key) throws IOException;
+
+ public void update(NutchDocument doc) throws IOException;
+
+ public void commit() throws IOException;
+
+ public void close() throws IOException;
+
+ /** Returns a String describing the IndexWriter instance and the specific parameters it can take */
+ public String describe();
+}
+
Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.ExtensionPoint;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.util.ObjectCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Creates and caches {@link IndexWriter} implementing plugins. */
+public class IndexWriters {
+
+ public final static Logger LOG = LoggerFactory
+ .getLogger(IndexWriters.class);
+
+ private IndexWriter[] indexWriters;
+
+ public IndexWriters(Configuration conf) {
+ ObjectCache objectCache = ObjectCache.get(conf);
+ synchronized (objectCache) {
+ this.indexWriters = (IndexWriter[]) objectCache
+ .getObject(IndexWriter.class.getName());
+ if (this.indexWriters == null) {
+ try {
+ ExtensionPoint point = PluginRepository.get(conf)
+ .getExtensionPoint(IndexWriter.X_POINT_ID);
+ if (point == null)
+ throw new RuntimeException(IndexWriter.X_POINT_ID
+ + " not found.");
+ Extension[] extensions = point.getExtensions();
+ HashMap<String, IndexWriter> indexerMap = new HashMap<String, IndexWriter>();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ IndexWriter writer = (IndexWriter) extension
+ .getExtensionInstance();
+ LOG.info("Adding " + writer.getClass().getName());
+ if (!indexerMap.containsKey(writer.getClass().getName())) {
+ indexerMap.put(writer.getClass().getName(), writer);
+ }
+ }
+ objectCache.setObject(IndexWriter.class.getName(), indexerMap
+ .values().toArray(new IndexWriter[0]));
+ } catch (PluginRuntimeException e) {
+ throw new RuntimeException(e);
+ }
+ this.indexWriters = (IndexWriter[]) objectCache
+ .getObject(IndexWriter.class.getName());
+ }
+ }
+ }
+
+ public void open(JobConf job, String name) throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].open(job, name);
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ public void write(NutchDocument doc) throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].write(doc);
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ public void update(NutchDocument doc) throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].update(doc);
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ public void delete(String key) throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].delete(key);
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ public void close() throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].close();
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ public void commit() throws IOException {
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ try {
+ this.indexWriters[i].commit();
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ }
+ }
+
+ // lists the active IndexWriters and their configuration
+ public String describe() throws IOException {
+ StringBuffer buffer = new StringBuffer();
+ if (this.indexWriters.length == 0)
+ buffer.append("No IndexWriters activated - check your configuration\n");
+ else
+ buffer.append("Active IndexWriters :\n");
+ for (int i = 0; i < this.indexWriters.length; i++) {
+ buffer.append(this.indexWriters[i].describe()).append("\n");
+ }
+ return buffer.toString();
+ }
+
+}
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java Thu Mar 7 11:19:24 2013
@@ -56,6 +56,7 @@ implements Mapper<Text, Writable, Text,
public static final Logger LOG = LoggerFactory.getLogger(IndexerMapReduce.class);
+ public static final String INDEXER_PARAMS = "indexer.additional.params";
public static final String INDEXER_DELETE = "indexer.delete";
public static final String INDEXER_DELETE_ROBOTS_NOINDEX = "indexer.delete.robots.noindex";
public static final String INDEXER_SKIP_NOTMODIFIED = "indexer.skip.notmodified";
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java Thu Mar 7 11:19:24 2013
@@ -26,39 +26,32 @@ import org.apache.hadoop.mapred.RecordWr
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-public class IndexerOutputFormat extends FileOutputFormat<Text, NutchIndexAction> {
+public class IndexerOutputFormat extends
+ FileOutputFormat<Text, NutchIndexAction> {
- @Override
- public RecordWriter<Text, NutchIndexAction> getRecordWriter(FileSystem ignored,
- JobConf job, String name, Progressable progress) throws IOException {
-
- // populate JobConf with field indexing options
- IndexingFilters filters = new IndexingFilters(job);
-
- final NutchIndexWriter[] writers =
- NutchIndexWriterFactory.getNutchIndexWriters(job);
+ @Override
+ public RecordWriter<Text, NutchIndexAction> getRecordWriter(
+ FileSystem ignored, JobConf job, String name, Progressable progress)
+ throws IOException {
- for (final NutchIndexWriter writer : writers) {
- writer.open(job, name);
- }
- return new RecordWriter<Text, NutchIndexAction>() {
+ final IndexWriters writers = new IndexWriters(job);
+
+ writers.open(job, name);
+
+ return new RecordWriter<Text, NutchIndexAction>() {
- public void close(Reporter reporter) throws IOException {
- for (final NutchIndexWriter writer : writers) {
- writer.close();
- }
- }
-
- public void write(Text key, NutchIndexAction indexAction) throws IOException {
- for (final NutchIndexWriter writer : writers) {
- if (indexAction.action == NutchIndexAction.ADD) {
- writer.write(indexAction.doc);
- }
- if (indexAction.action == NutchIndexAction.DELETE) {
- writer.delete(key.toString());
- }
- }
- }
- };
- }
+ public void close(Reporter reporter) throws IOException {
+ writers.close();
+ }
+
+ public void write(Text key, NutchIndexAction indexAction)
+ throws IOException {
+ if (indexAction.action == NutchIndexAction.ADD) {
+ writers.write(indexAction.doc);
+ } else if (indexAction.action == NutchIndexAction.DELETE) {
+ writers.delete(key.toString());
+ }
+ }
+ };
+ }
}
Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.indexer.solr.SolrConstants;
+import org.apache.nutch.util.HadoopFSUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.TimingUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Generic indexer which relies on the plugins implementing IndexWriter
+ **/
+
+public class IndexingJob extends Configured implements Tool {
+
+ public static Logger LOG = LoggerFactory.getLogger(IndexingJob.class);
+
+ public IndexingJob() {
+ super(null);
+ }
+
+ public IndexingJob(Configuration conf) {
+ super(conf);
+ }
+
+ public void index(Path crawlDb, Path linkDb, List<Path> segments)
+ throws IOException {
+ // SOLR SPECIFIC COMMIT PARAM USED BY CRAWL CLASS
+ boolean noCommit = !getConf().getBoolean(SolrConstants.COMMIT_INDEX,
+ true);
+ index(crawlDb, linkDb, segments, noCommit, false, null);
+ }
+
+ public void index(Path crawlDb, Path linkDb, List<Path> segments,
+ boolean noCommit) throws IOException {
+ index(crawlDb, linkDb, segments, noCommit, false, null);
+ }
+
+ public void index(Path crawlDb, Path linkDb, List<Path> segments,
+ boolean noCommit, boolean deleteGone) throws IOException {
+ index(crawlDb, linkDb, segments, noCommit, deleteGone, null);
+ }
+
+ public void index(Path crawlDb, Path linkDb, List<Path> segments,
+ boolean noCommit, boolean deleteGone, String params)
+ throws IOException {
+ index(crawlDb, linkDb, segments, noCommit, deleteGone, params, false,
+ false);
+ }
+
+ public void index(Path crawlDb, Path linkDb, List<Path> segments,
+ boolean noCommit, boolean deleteGone, String params,
+ boolean filter, boolean normalize) throws IOException {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ long start = System.currentTimeMillis();
+ LOG.info("Indexer: starting at " + sdf.format(start));
+
+ final JobConf job = new NutchJob(getConf());
+ job.setJobName("Indexer");
+
+ LOG.info("Indexer: deleting gone documents: " + deleteGone);
+ LOG.info("Indexer: URL filtering: " + filter);
+ LOG.info("Indexer: URL normalizing: " + normalize);
+
+ IndexWriters writers = new IndexWriters(getConf());
+ LOG.info(writers.describe());
+
+ IndexerMapReduce.initMRJob(crawlDb, linkDb, segments, job);
+
+ // NOW PASSED ON THE COMMAND LINE AS A HADOOP PARAM
+ // job.set(SolrConstants.SERVER_URL, solrUrl);
+
+ job.setBoolean(IndexerMapReduce.INDEXER_DELETE, deleteGone);
+ job.setBoolean(IndexerMapReduce.URL_FILTERING, filter);
+ job.setBoolean(IndexerMapReduce.URL_NORMALIZING, normalize);
+
+ if (params != null) {
+ job.set(IndexerMapReduce.INDEXER_PARAMS, params);
+ }
+
+ job.setReduceSpeculativeExecution(false);
+
+ final Path tmp = new Path("tmp_" + System.currentTimeMillis() + "-"
+ + new Random().nextInt());
+
+ FileOutputFormat.setOutputPath(job, tmp);
+ try {
+ JobClient.runJob(job);
+ // do the commits once and for all the reducers in one go
+ if (!noCommit) {
+ writers.open(job,"commit");
+ writers.commit();
+ }
+ long end = System.currentTimeMillis();
+ LOG.info("Indexer: finished at " + sdf.format(end) + ", elapsed: "
+ + TimingUtil.elapsedTime(start, end));
+ } finally {
+ FileSystem.get(job).delete(tmp, true);
+ }
+ }
+
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err
+ .println("Usage: Indexer <crawldb> [-linkdb <linkdb>] [-params k1=v1&k2=v2...] (<segment> ... | -dir <segments>) [-noCommit] [-deleteGone] [-filter] [-normalize]");
+ IndexWriters writers = new IndexWriters(getConf());
+ System.err.println(writers.describe());
+ return -1;
+ }
+
+ final Path crawlDb = new Path(args[0]);
+ Path linkDb = null;
+
+ final List<Path> segments = new ArrayList<Path>();
+ String params = null;
+
+ boolean noCommit = false;
+ boolean deleteGone = false;
+ boolean filter = false;
+ boolean normalize = false;
+
+ for (int i = 1; i < args.length; i++) {
+ if (args[i].equals("-linkdb")) {
+ linkDb = new Path(args[++i]);
+ } else if (args[i].equals("-dir")) {
+ Path dir = new Path(args[++i]);
+ FileSystem fs = dir.getFileSystem(getConf());
+ FileStatus[] fstats = fs.listStatus(dir,
+ HadoopFSUtil.getPassDirectoriesFilter(fs));
+ Path[] files = HadoopFSUtil.getPaths(fstats);
+ for (Path p : files) {
+ segments.add(p);
+ }
+ } else if (args[i].equals("-noCommit")) {
+ noCommit = true;
+ } else if (args[i].equals("-deleteGone")) {
+ deleteGone = true;
+ } else if (args[i].equals("-filter")) {
+ filter = true;
+ } else if (args[i].equals("-normalize")) {
+ normalize = true;
+ } else if (args[i].equals("-params")) {
+ params = args[++i];
+ } else {
+ segments.add(new Path(args[i]));
+ }
+ }
+
+ try {
+ index(crawlDb, linkDb, segments, noCommit, deleteGone, params,
+ filter, normalize);
+ return 0;
+ } catch (final Exception e) {
+ LOG.error("Indexer: " + StringUtils.stringifyException(e));
+ return -1;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ final int res = ToolRunner.run(NutchConfiguration.create(),
+ new IndexingJob(), args);
+ System.exit(res);
+ }
+}
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java Thu Mar 7 11:19:24 2013
@@ -29,7 +29,7 @@ import java.net.MalformedURLException;
public class SolrUtils {
- public static Logger LOG = LoggerFactory.getLogger(SolrIndexer.class);
+ public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class);
public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException {
HttpClient client=new HttpClient();
Modified: nutch/trunk/src/plugin/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/build.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/plugin/build.xml (original)
+++ nutch/trunk/src/plugin/build.xml Thu Mar 7 11:19:24 2013
@@ -34,6 +34,7 @@
<ant dir="index-more" target="deploy"/>
<ant dir="index-static" target="deploy"/>
<ant dir="index-metadata" target="deploy"/>
+ <ant dir="indexer-solr" target="deploy"/>
<ant dir="language-identifier" target="deploy"/>
<ant dir="lib-http" target="deploy"/>
<ant dir="lib-nekohtml" target="deploy"/>
@@ -118,6 +119,7 @@
<ant dir="index-more" target="clean"/>
<ant dir="index-static" target="clean"/>
<ant dir="index-metadata" target="clean"/>
+ <ant dir="indexer-solr" target="clean"/>
<ant dir="language-identifier" target="clean"/>
<ant dir="lib-commons-httpclient" target="clean"/>
<ant dir="lib-http" target="clean"/>
Added: nutch/trunk/src/plugin/indexer-solr/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/build.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/build.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/build.xml Thu Mar 7 11:19:24 2013
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="indexer-solr" default="jar-core">
+
+ <import file="../build-plugin.xml" />
+
+</project>
Added: nutch/trunk/src/plugin/indexer-solr/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/ivy.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/ivy.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/ivy.xml Thu Mar 7 11:19:24 2013
@@ -0,0 +1,43 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../..//ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ <dependency org="org.apache.solr" name="solr-solrj" rev="3.4.0"
+ conf="*->default"/>
+ </dependencies>
+
+</ivy-module>
Added: nutch/trunk/src/plugin/indexer-solr/plugin.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/plugin.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/plugin.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/plugin.xml Thu Mar 7 11:19:24 2013
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin id="indexer-solr" name="SOLRIndexWriter" version="1.0.0"
+ provider-name="nutch.apache.org">
+
+ <runtime>
+ <library name="indexer-solr.jar">
+ <export name="*" />
+ </library>
+
+ <library name="activation-1.1.jar"/>
+ <library name="commons-codec-1.4.jar"/>
+ <library name="commons-httpclient-3.1.jar"/>
+ <library name="commons-io-1.4.jar"/>
+ <library name="commons-logging-1.1.1.jar"/>
+ <library name="geronimo-stax-api_1.0_spec-1.0.1.jar"/>
+ <library name="jline-0.9.1.jar"/>
+ <library name="log4j-1.2.15.jar"/>
+ <library name="lucene-core-3.4.0.jar"/>
+ <library name="mail-1.4.1.jar"/>
+ <library name="slf4j-api-1.6.1.jar"/>
+ <library name="solr-solrj-3.4.0.jar"/>
+ <library name="stax-api-1.0.1.jar"/>
+ <library name="wstx-asl-3.2.7.jar"/>
+ <library name="zookeeper-3.3.1.jar"/>
+ </runtime>
+
+ <requires>
+ <import plugin="nutch-extensionpoints" />
+ </requires>
+
+ <extension id="org.apache.nutch.indexer.solr"
+ name="SOLR Index Writer"
+ point="org.apache.nutch.indexer.IndexWriter">
+ <implementation id="SOLRIndexWriter"
+ class="org.apache.nutch.indexwriter.solr.SolrIndexWriter" />
+ </extension>
+
+</plugin>
Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+public interface SolrConstants {
+ public static final String SOLR_PREFIX = "solr.";
+
+ public static final String SERVER_URL = SOLR_PREFIX + "server.url";
+
+ public static final String COMMIT_SIZE = SOLR_PREFIX + "commit.size";
+
+ public static final String MAPPING_FILE = SOLR_PREFIX + "mapping.file";
+
+ public static final String USE_AUTH = SOLR_PREFIX + "auth";
+
+ public static final String USERNAME = SOLR_PREFIX + "auth.username";
+
+ public static final String PASSWORD = SOLR_PREFIX + "auth.password";
+
+ @Deprecated
+ public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index";
+
+ @Deprecated
+ public static final String PARAMS = SOLR_PREFIX + "params";
+
+}
Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.IndexWriter;
+import org.apache.nutch.indexer.IndexerMapReduce;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.NutchField;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.DateUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrIndexWriter implements IndexWriter {
+
+ public static final Logger LOG = LoggerFactory
+ .getLogger(SolrIndexWriter.class);
+
+ private SolrServer solr;
+ private SolrMappingReader solrMapping;
+ private ModifiableSolrParams params;
+
+ private Configuration config;
+
+ private final List<SolrInputDocument> inputDocs = new ArrayList<SolrInputDocument>();
+
+ private int batchSize;
+ private int numDeletes = 0;
+ private boolean delete = false;
+
+ public void open(JobConf job, String name) throws IOException {
+ SolrServer server = SolrUtils.getCommonsHttpSolrServer(job);
+ init(server, job);
+ }
+
+ // package protected for tests
+ void init(SolrServer server, JobConf job) throws IOException {
+ solr = server;
+ batchSize = job.getInt(SolrConstants.COMMIT_SIZE, 1000);
+ solrMapping = SolrMappingReader.getInstance(job);
+ delete = job.getBoolean(IndexerMapReduce.INDEXER_DELETE, false);
+ // parse optional params
+ params = new ModifiableSolrParams();
+ String paramString = job.get(IndexerMapReduce.INDEXER_PARAMS);
+ if (paramString != null) {
+ String[] values = paramString.split("&");
+ for (String v : values) {
+ String[] kv = v.split("=");
+ if (kv.length < 2) {
+ continue;
+ }
+ params.add(kv[0], kv[1]);
+ }
+ }
+ }
+
+ public void delete(String key) throws IOException {
+ if (delete) {
+ try {
+ solr.deleteById(key);
+ numDeletes++;
+ } catch (final SolrServerException e) {
+ throw makeIOException(e);
+ }
+ }
+ }
+
+ @Override
+ public void update(NutchDocument doc) throws IOException {
+ write(doc);
+ }
+
+ public void write(NutchDocument doc) throws IOException {
+ final SolrInputDocument inputDoc = new SolrInputDocument();
+ for (final Entry<String, NutchField> e : doc) {
+ for (final Object val : e.getValue().getValues()) {
+ // normalise the string representation for a Date
+ Object val2 = val;
+
+ if (val instanceof Date) {
+ val2 = DateUtil.getThreadLocalDateFormat().format(val);
+ }
+
+ if (e.getKey().equals("content") || e.getKey().equals("title")) {
+ val2 = SolrUtils.stripNonCharCodepoints((String) val);
+ }
+
+ inputDoc.addField(solrMapping.mapKey(e.getKey()), val2, e
+ .getValue().getWeight());
+ String sCopy = solrMapping.mapCopyKey(e.getKey());
+ if (sCopy != e.getKey()) {
+ inputDoc.addField(sCopy, val);
+ }
+ }
+ }
+
+ inputDoc.setDocumentBoost(doc.getWeight());
+ inputDocs.add(inputDoc);
+ if (inputDocs.size() + numDeletes >= batchSize) {
+ try {
+ LOG.info("Indexing " + Integer.toString(inputDocs.size())
+ + " documents");
+ LOG.info("Deleting " + Integer.toString(numDeletes)
+ + " documents");
+ numDeletes = 0;
+ UpdateRequest req = new UpdateRequest();
+ req.add(inputDocs);
+ req.setParams(params);
+ req.process(solr);
+ } catch (final SolrServerException e) {
+ throw makeIOException(e);
+ }
+ inputDocs.clear();
+ }
+ }
+
+ public void close() throws IOException {
+ try {
+ if (!inputDocs.isEmpty()) {
+ LOG.info("Indexing " + Integer.toString(inputDocs.size())
+ + " documents");
+ if (numDeletes > 0) {
+ LOG.info("Deleting " + Integer.toString(numDeletes)
+ + " documents");
+ }
+ UpdateRequest req = new UpdateRequest();
+ req.add(inputDocs);
+ req.setParams(params);
+ req.process(solr);
+ inputDocs.clear();
+ }
+ } catch (final SolrServerException e) {
+ throw makeIOException(e);
+ }
+ }
+
+ @Override
+ public void commit() throws IOException {
+ try {
+ solr.commit();
+ } catch (SolrServerException e) {
+ throw makeIOException(e);
+ }
+ }
+
+ public static IOException makeIOException(SolrServerException e) {
+ final IOException ioe = new IOException();
+ ioe.initCause(e);
+ return ioe;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return config;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ config = conf;
+ String serverURL = conf.get(SolrConstants.SERVER_URL);
+ if (serverURL == null) {
+ String message = "Missing SOLR URL. Should be set via -D "
+ + SolrConstants.SERVER_URL;
+ message+="\n"+describe();
+ LOG.error(message);
+ throw new RuntimeException(message);
+ }
+ }
+
+ public String describe(){
+ StringBuffer sb = new StringBuffer("SOLRIndexWriter\n");
+ sb.append("\t").append(SolrConstants.SERVER_URL).append(" : URL of the SOLR instance (mandatory)\n");
+ sb.append("\t").append(SolrConstants.COMMIT_SIZE).append(" : buffer size when sending to SOLR (default 1000)\n");
+ sb.append("\t").append(SolrConstants.MAPPING_FILE).append(" : name of the mapping file for fields (default solrindex-mapping.xml)\n");
+ sb.append("\t").append(SolrConstants.USE_AUTH).append(" : use authentication (default false)\n");
+ sb.append("\t").append(SolrConstants.USERNAME).append(" : use authentication (default false)\n");
+ sb.append("\t").append(SolrConstants.USE_AUTH).append(" : username for authentication\n");
+ sb.append("\t").append(SolrConstants.PASSWORD).append(" : password for authentication\n");
+ return sb.toString();
+ }
+
+}
Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.ObjectCache;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+public class SolrMappingReader {
+ public static Logger LOG = LoggerFactory.getLogger(SolrMappingReader.class);
+
+ private Configuration conf;
+
+ private Map<String, String> keyMap = new HashMap<String, String>();
+ private Map<String, String> copyMap = new HashMap<String, String>();
+ private String uniqueKey = "id";
+
+ public static synchronized SolrMappingReader getInstance(Configuration conf) {
+ ObjectCache cache = ObjectCache.get(conf);
+ SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
+ if (instance == null) {
+ instance = new SolrMappingReader(conf);
+ cache.setObject(SolrMappingReader.class.getName(), instance);
+ }
+ return instance;
+ }
+
+ protected SolrMappingReader(Configuration conf) {
+ this.conf = conf;
+ parseMapping();
+ }
+
+ private void parseMapping() {
+ InputStream ssInputStream = null;
+ ssInputStream = conf.getConfResourceAsInputStream(conf.get(SolrConstants.MAPPING_FILE, "solrindex-mapping.xml"));
+
+ InputSource inputSource = new InputSource(ssInputStream);
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document document = builder.parse(inputSource);
+ Element rootElement = document.getDocumentElement();
+ NodeList fieldList = rootElement.getElementsByTagName("field");
+ if (fieldList.getLength() > 0) {
+ for (int i = 0; i < fieldList.getLength(); i++) {
+ Element element = (Element) fieldList.item(i);
+ LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+ keyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+ }
+ }
+ NodeList copyFieldList = rootElement.getElementsByTagName("copyField");
+ if (copyFieldList.getLength() > 0) {
+ for (int i = 0; i < copyFieldList.getLength(); i++) {
+ Element element = (Element) copyFieldList.item(i);
+ LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+ copyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+ }
+ }
+ NodeList uniqueKeyItem = rootElement.getElementsByTagName("uniqueKey");
+ if (uniqueKeyItem.getLength() > 1) {
+ LOG.warn("More than one unique key definitions found in solr index mapping, using default 'id'");
+ uniqueKey = "id";
+ }
+ else if (uniqueKeyItem.getLength() == 0) {
+ LOG.warn("No unique key definition found in solr index mapping using, default 'id'");
+ }
+ else{
+ uniqueKey = uniqueKeyItem.item(0).getFirstChild().getNodeValue();
+ }
+ } catch (MalformedURLException e) {
+ LOG.warn(e.toString());
+ } catch (SAXException e) {
+ LOG.warn(e.toString());
+ } catch (IOException e) {
+ LOG.warn(e.toString());
+ } catch (ParserConfigurationException e) {
+ LOG.warn(e.toString());
+ }
+ }
+
+ public Map<String, String> getKeyMap() {
+ return keyMap;
+ }
+
+ public Map<String, String> getCopyMap() {
+ return copyMap;
+ }
+
+ public String getUniqueKey() {
+ return uniqueKey;
+ }
+
+ public String hasCopy(String key) {
+ if (copyMap.containsKey(key)) {
+ key = (String) copyMap.get(key);
+ }
+ return key;
+ }
+
+ public String mapKey(String key) throws IOException {
+ if(keyMap.containsKey(key)) {
+ key = (String) keyMap.get(key);
+ }
+ return key;
+ }
+
+ public String mapCopyKey(String key) throws IOException {
+ if(copyMap.containsKey(key)) {
+ key = (String) copyMap.get(key);
+ }
+ return key;
+ }
+}
Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java Thu Mar 7 11:19:24 2013
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.params.HttpClientParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+
+import java.net.MalformedURLException;
+
+public class SolrUtils {
+
+ public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class);
+
+ public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException {
+ HttpClient client=new HttpClient();
+
+ // Check for username/password
+ if (job.getBoolean(SolrConstants.USE_AUTH, false)) {
+ String username = job.get(SolrConstants.USERNAME);
+
+ LOG.info("Authenticating as: " + username);
+
+ AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME);
+
+ client.getState().setCredentials(scope, new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD)));
+
+ HttpClientParams params = client.getParams();
+ params.setAuthenticationPreemptive(true);
+
+ client.setParams(params);
+ }
+
+ String serverURL = job.get(SolrConstants.SERVER_URL);
+
+ return new CommonsHttpSolrServer(serverURL, client);
+ }
+
+ public static String stripNonCharCodepoints(String input) {
+ StringBuilder retval = new StringBuilder();
+ char ch;
+
+ for (int i = 0; i < input.length(); i++) {
+ ch = input.charAt(i);
+
+ // Strip all non-characters http://unicode.org/cldr/utility/list-unicodeset.jsp?a=[:Noncharacter_Code_Point=True:]
+ // and non-printable control characters except tabulator, new line and carriage return
+ if (ch % 0x10000 != 0xffff && // 0xffff - 0x10ffff range step 0x10000
+ ch % 0x10000 != 0xfffe && // 0xfffe - 0x10fffe range
+ (ch <= 0xfdd0 || ch >= 0xfdef) && // 0xfdd0 - 0xfdef
+ (ch > 0x1F || ch == 0x9 || ch == 0xa || ch == 0xd)) {
+
+ retval.append(ch);
+ }
+ }
+
+ return retval.toString();
+ }
+}
Modified: nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml (original)
+++ nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml Thu Mar 7 11:19:24 2013
@@ -29,6 +29,10 @@
name="Nutch Indexing Filter"/>
<extension-point
+ id="org.apache.nutch.indexer.IndexWriter"
+ name="Nutch Index Writer"/>
+
+<extension-point
id="org.apache.nutch.parse.Parser"
name="Nutch Content Parser"/>