You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2013/03/07 12:19:25 UTC

svn commit: r1453776 - in /nutch/trunk: ./ conf/ ivy/ src/bin/ src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/indexer/ src/java/org/apache/nutch/indexer/solr/ src/plugin/ src/plugin/indexer-solr/ src/plugin/indexer-solr/src/ src/plugin/inde...

Author: jnioche
Date: Thu Mar  7 11:19:24 2013
New Revision: 1453776

URL: http://svn.apache.org/r1453776
Log:
NUTCH-1047 Pluggable indexing backends

Added:
    nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java
    nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java
    nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java
    nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java
    nutch/trunk/src/plugin/indexer-solr/
    nutch/trunk/src/plugin/indexer-solr/build.xml
    nutch/trunk/src/plugin/indexer-solr/ivy.xml
    nutch/trunk/src/plugin/indexer-solr/plugin.xml
    nutch/trunk/src/plugin/indexer-solr/src/
    nutch/trunk/src/plugin/indexer-solr/src/java/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java
    nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
Removed:
    nutch/trunk/src/java/org/apache/nutch/indexer/NutchIndexWriter.java
    nutch/trunk/src/java/org/apache/nutch/indexer/NutchIndexWriterFactory.java
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrClean.java
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/build.xml
    nutch/trunk/conf/nutch-default.xml
    nutch/trunk/default.properties
    nutch/trunk/ivy/ivy.xml
    nutch/trunk/src/bin/nutch
    nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
    nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
    nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java
    nutch/trunk/src/plugin/build.xml
    nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml

Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Mar  7 11:19:24 2013
@@ -2,6 +2,8 @@ Nutch Change Log
 
 (trunk): Current Development
 
+* NUTCH-1047 Pluggable indexing backends (jnioche)
+
 * NUTCH-1536 Ant build file has hardcoded conf dir location (zm via lewismc)
 
 * NUTCH-1420 Get rid of the dreaded � (markus via lewismc)

Modified: nutch/trunk/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/build.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/build.xml (original)
+++ nutch/trunk/build.xml Thu Mar  7 11:19:24 2013
@@ -133,7 +133,7 @@
           
 	<!-- build the main artifact -->
 	<jar jarfile="${maven-jar}" basedir="${build.classes}" />
-
+  	
 	<!-- build the javadoc artifact -->
     <javadoc
       destdir="${release.dir}/javadoc"
@@ -157,6 +157,7 @@
       <packageset dir="${plugins.dir}/index-metadata/src/java"/>
       <packageset dir="${plugins.dir}/index-more/src/java"/>
       <packageset dir="${plugins.dir}/index-static/src/java"/>
+      <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
       <packageset dir="${plugins.dir}/language-identifier/src/java"/>
       <packageset dir="${plugins.dir}/lib-http/src/java"/>
       <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -208,6 +209,7 @@
       <group title="Scoring Plugins" packages="${plugins.scoring}"/>
       <group title="Parse Plugins" packages="${plugins.parse}"/>
       <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
+      <group title="Indexer Plugins" packages="${plugins.indexer}"/>
       <group title="Misc. Plugins" packages="${plugins.misc}"/>
     </javadoc>	
 	<jar jarfile="${maven-javadoc-jar}">
@@ -545,6 +547,7 @@
       <packageset dir="${plugins.dir}/index-metadata/src/java"/>
       <packageset dir="${plugins.dir}/index-more/src/java"/>
       <packageset dir="${plugins.dir}/index-static/src/java"/>
+      <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
       <packageset dir="${plugins.dir}/language-identifier/src/java"/>
       <packageset dir="${plugins.dir}/lib-http/src/java"/>
       <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -594,6 +597,7 @@
       <group title="Scoring Plugins" packages="${plugins.scoring}"/>
       <group title="Parse Plugins" packages="${plugins.parse}"/>
       <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
+      <group title="Indexer Plugins" packages="${plugins.indexer}"/>
       <group title="Misc. Plugins" packages="${plugins.misc}"/>
     </javadoc>
     <!-- Copy the plugin.dtd file to the plugin doc-files dir -->

Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Thu Mar  7 11:19:24 2013
@@ -990,7 +990,7 @@
 
 <property>
   <name>plugin.includes</name>
-  <value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|scoring-opic|urlnormalizer-(pass|regex|basic)</value>
+  <value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)</value>
   <description>Regular expression naming plugin directory names to
   include.  Any plugin not matching this expression is excluded.
   In any case you need at least include the nutch-extensionpoints plugin. By

Modified: nutch/trunk/default.properties
URL: http://svn.apache.org/viewvc/nutch/trunk/default.properties?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/default.properties (original)
+++ nutch/trunk/default.properties Thu Mar  7 11:19:24 2013
@@ -150,6 +150,12 @@ plugins.index=\
    org.apache.nutch.indexer.urlmeta*
 
 #
+# Indexing Backend Plugins
+#
+plugins.indexer=\
+   org.apache.nutch.indexwriter.solr*
+
+#
 # Misc. Plugins
 #
 # (gathers plugins that cannot be dispatched

Modified: nutch/trunk/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Thu Mar  7 11:19:24 2013
@@ -13,10 +13,14 @@
 
 <ivy-module version="1.0">
 	<info organisation="org.apache.nutch" module="nutch">
-		<license name="Apache 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt/"/>
+		<license name="Apache 2.0"
+			url="http://www.apache.org/licenses/LICENSE-2.0.txt/" />
 		<ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org" />
-		<description homepage="http://nutch.apache.org">Nutch is an open source web-search software. It builds on 
-		Hadoop, Tika and Solr, adding web-specifics, such as a crawler, a link-graph database etc.
+		<description homepage="http://nutch.apache.org">Nutch is an open source web-search
+			software. It builds on
+			Hadoop, Tika and Solr, adding web-specifics,
+			such as a crawler, a link-graph
+			database etc.
 		</description>
 	</info>
 
@@ -31,8 +35,12 @@
 
 	<dependencies>
 		<dependency org="org.apache.solr" name="solr-solrj" rev="3.4.0"
-			conf="*->default"/>
-		<dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.1" conf="*->master" />
+			conf="*->default" />
+
+		<dependency org="org.slf4j" name="slf4j-api" rev="1.6.1"
+			conf="*->master" />
+		<dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.1"
+			conf="*->master" />
 
 		<dependency org="commons-lang" name="commons-lang" rev="2.6"
 			conf="*->default" />
@@ -55,15 +63,16 @@
 
 		<dependency org="org.apache.tika" name="tika-core" rev="1.2" />
 		<dependency org="com.ibm.icu" name="icu4j" rev="4.0.1" />
-		
-		<dependency org="org.mortbay.jetty" name="jetty-client" rev="6.1.22" />
+
+		<dependency org="org.mortbay.jetty" name="jetty-client"
+			rev="6.1.22" />
 
 		<dependency org="log4j" name="log4j" rev="1.2.15" conf="*->master" />
 
 		<dependency org="xerces" name="xercesImpl" rev="2.9.1" />
 		<dependency org="xerces" name="xmlParserAPIs" rev="2.6.2" />
 		<dependency org="oro" name="oro" rev="2.0.8" />
-		
+
 		<dependency org="com.google.guava" name="guava" rev="11.0.2" />
 
 		<!--Configuration: test -->
@@ -78,10 +87,10 @@
 		<dependency org="org.mortbay.jetty" name="jetty-util" rev="6.1.22"
 			conf="test->default" />
 
-                <!--global exclusion-->
-             	<exclude module="jmxtools" />
-             	<exclude module="jms" />
-             	<exclude module="jmxri" />
+		<!--global exclusion -->
+		<exclude module="jmxtools" />
+		<exclude module="jms" />
+		<exclude module="jmxri" />
 
 	</dependencies>
 

Modified: nutch/trunk/src/bin/nutch
URL: http://svn.apache.org/viewvc/nutch/trunk/src/bin/nutch?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/bin/nutch (original)
+++ nutch/trunk/src/bin/nutch Thu Mar  7 11:19:24 2013
@@ -61,9 +61,11 @@ if [ $# = 0 ]; then
   echo "  updatedb          update crawl db from segments after fetching"
   echo "  invertlinks       create a linkdb from parsed segments"
   echo "  mergelinkdb       merge linkdb-s, with optional filtering"
+  echo "  index             run the plugin-based indexer on parsed segments and linkdb"
   echo "  solrindex         run the solr indexer on parsed segments and linkdb"
   echo "  solrdedup         remove duplicates from solr"
   echo "  solrclean         remove HTTP 301 and 404 documents from solr"
+  echo "  clean             remove HTTP 301 and 404 documents from indexing backends configured via plugins"
   echo "  parsechecker      check the parser for a given url"
   echo "  indexchecker      check the indexing filters for a given url"
   echo "  domainstats       calculate domain statistics from crawldb"
@@ -219,11 +221,17 @@ elif [ "$COMMAND" = "invertlinks" ] ; th
 elif [ "$COMMAND" = "mergelinkdb" ] ; then
   CLASS=org.apache.nutch.crawl.LinkDbMerger
 elif [ "$COMMAND" = "solrindex" ] ; then
-  CLASS=org.apache.nutch.indexer.solr.SolrIndexer
+  CLASS="org.apache.nutch.indexer.IndexingJob -D solr.server.url=$1"
+  shift
+elif [ "$COMMAND" = "index" ] ; then
+  CLASS=org.apache.nutch.indexer.IndexingJob
 elif [ "$COMMAND" = "solrdedup" ] ; then
   CLASS=org.apache.nutch.indexer.solr.SolrDeleteDuplicates
 elif [ "$COMMAND" = "solrclean" ] ; then
-  CLASS=org.apache.nutch.indexer.solr.SolrClean
+  CLASS="org.apache.nutch.indexer.CleaningJob -D solr.server.url=$2 $1"
+  shift; shift
+elif [ "$COMMAND" = "clean" ] ; then
+  CLASS=org.apache.nutch.indexer.CleaningJob
 elif [ "$COMMAND" = "parsechecker" ] ; then
   CLASS=org.apache.nutch.parse.ParserChecker
 elif [ "$COMMAND" = "indexchecker" ] ; then

Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java Thu Mar  7 11:19:24 2013
@@ -31,8 +31,8 @@ import org.apache.hadoop.mapred.*;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.nutch.parse.ParseSegment;
+import org.apache.nutch.indexer.IndexingJob;
 import org.apache.nutch.indexer.solr.SolrDeleteDuplicates;
-import org.apache.nutch.indexer.solr.SolrIndexer;
 import org.apache.nutch.util.HadoopFSUtil;
 import org.apache.nutch.util.NutchConfiguration;
 import org.apache.nutch.util.NutchJob;
@@ -96,6 +96,11 @@ public class Crawl extends Configured im
     if (solrUrl == null) {
       LOG.warn("solrUrl is not set, indexing will be skipped...");
     }
+    else {
+        // for simplicity assume that SOLR is used 
+        // and pass its URL via conf 
+        getConf().set("solr.server.url", solrUrl);
+    }
 
     FileSystem fs = FileSystem.get(job);
 
@@ -145,9 +150,11 @@ public class Crawl extends Configured im
       if (solrUrl != null) {
         // index, dedup & merge
         FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
-        SolrIndexer indexer = new SolrIndexer(getConf());
-        indexer.indexSolr(solrUrl, crawlDb, linkDb, 
-          Arrays.asList(HadoopFSUtil.getPaths(fstats)));
+        
+        IndexingJob indexer = new IndexingJob(getConf());
+        indexer.index(crawlDb, linkDb, 
+                Arrays.asList(HadoopFSUtil.getPaths(fstats)));
+
         SolrDeleteDuplicates dedup = new SolrDeleteDuplicates();
         dedup.setConf(getConf());
         dedup.dedup(solrUrl);

Added: nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/CleaningJob.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.CrawlDb;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.TimingUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The class scans CrawlDB looking for entries with status DB_GONE (404) and
+ * sends delete requests to indexers for those documents.
+ * 
+ */
+
+public class CleaningJob implements Tool {
+    public static final Logger LOG = LoggerFactory.getLogger(CleaningJob.class);
+    private Configuration conf;
+
+    @Override
+    public Configuration getConf() {
+        return conf;
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        this.conf = conf;
+    }
+
+    public static class DBFilter implements
+            Mapper<Text, CrawlDatum, ByteWritable, Text> {
+        private ByteWritable OUT = new ByteWritable(CrawlDatum.STATUS_DB_GONE);
+
+        @Override
+        public void configure(JobConf arg0) {
+        }
+
+        @Override
+        public void close() throws IOException {
+        }
+
+        @Override
+        public void map(Text key, CrawlDatum value,
+                OutputCollector<ByteWritable, Text> output, Reporter reporter)
+                throws IOException {
+
+            if (value.getStatus() == CrawlDatum.STATUS_DB_GONE) {
+                output.collect(OUT, key);
+            }
+        }
+    }
+
+    public static class DeleterReducer implements
+            Reducer<ByteWritable, Text, Text, ByteWritable> {
+        private static final int NUM_MAX_DELETE_REQUEST = 1000;
+        private int numDeletes = 0;
+        private int totalDeleted = 0;
+
+        private boolean noCommit = false;
+
+        IndexWriters writers = null;
+
+        @Override
+        public void configure(JobConf job) {
+            writers = new IndexWriters(job);
+            try {
+                writers.open(job, "Deletion");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            noCommit = job.getBoolean("noCommit", false);
+        }
+
+        @Override
+        public void close() throws IOException {
+            // BUFFERING OF CALLS TO INDEXER SHOULD BE HANDLED AT INDEXER LEVEL
+            // if (numDeletes > 0) {
+            // LOG.info("CleaningJob: deleting " + numDeletes + " documents");
+            // // TODO updateRequest.process(solr);
+            // totalDeleted += numDeletes;
+            // }
+
+            writers.close();
+
+            if (totalDeleted > 0 && !noCommit) {
+                writers.commit();
+            }
+
+            LOG.info("CleaningJob: deleted a total of " + totalDeleted
+                    + " documents");
+        }
+
+        @Override
+        public void reduce(ByteWritable key, Iterator<Text> values,
+                OutputCollector<Text, ByteWritable> output, Reporter reporter)
+                throws IOException {
+            while (values.hasNext()) {
+                Text document = values.next();
+                writers.delete(document.toString());
+                totalDeleted++;
+                reporter.incrCounter("CleaningJobStatus", "Deleted documents",
+                        1);
+                // if (numDeletes >= NUM_MAX_DELETE_REQUEST) {
+                // LOG.info("CleaningJob: deleting " + numDeletes
+                // + " documents");
+                // // TODO updateRequest.process(solr);
+                // // TODO updateRequest = new UpdateRequest();
+                // writers.delete(key.toString());
+                // totalDeleted += numDeletes;
+                // numDeletes = 0;
+                // }
+            }
+        }
+    }
+
+    public void delete(String crawldb, boolean noCommit) throws IOException {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        long start = System.currentTimeMillis();
+        LOG.info("CleaningJob: starting at " + sdf.format(start));
+
+        JobConf job = new NutchJob(getConf());
+
+        FileInputFormat.addInputPath(job, new Path(crawldb,
+                CrawlDb.CURRENT_NAME));
+        job.setBoolean("noCommit", noCommit);
+        job.setInputFormat(SequenceFileInputFormat.class);
+        job.setOutputFormat(NullOutputFormat.class);
+        job.setMapOutputKeyClass(ByteWritable.class);
+        job.setMapOutputValueClass(Text.class);
+        job.setMapperClass(DBFilter.class);
+        job.setReducerClass(DeleterReducer.class);
+
+        // need to expicitely allow deletions
+        job.setBoolean(IndexerMapReduce.INDEXER_DELETE, true);
+
+        JobClient.runJob(job);
+
+        long end = System.currentTimeMillis();
+        LOG.info("CleaningJob: finished at " + sdf.format(end) + ", elapsed: "
+                + TimingUtil.elapsedTime(start, end));
+    }
+
+    public int run(String[] args) throws IOException {
+        if (args.length < 1) {
+            System.err.println("Usage: CleaningJob <crawldb> [-noCommit]");
+            IndexWriters writers = new IndexWriters(getConf());
+            System.err.println(writers.describe());
+            return 1;
+        }
+
+        boolean noCommit = false;
+        if (args.length == 2 && args[1].equals("-noCommit")) {
+            noCommit = true;
+        }
+
+        try {
+            delete(args[0], noCommit);
+        } catch (final Exception e) {
+            LOG.error("CleaningJob: " + StringUtils.stringifyException(e));
+            System.err.println("ERROR CleaningJob: "
+                    + StringUtils.stringifyException(e));
+            return -1;
+        }
+        return 0;
+    }
+
+    public static void main(String[] args) throws Exception {
+        int result = ToolRunner.run(NutchConfiguration.create(),
+                new CleaningJob(), args);
+        System.exit(result);
+    }
+}

Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriter.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.plugin.Pluggable;
+
+public interface IndexWriter extends Pluggable, Configurable {
+    /** The name of the extension point. */
+    final static String X_POINT_ID = IndexWriter.class.getName();
+    
+    public void open(JobConf job, String name) throws IOException;
+
+    public void write(NutchDocument doc) throws IOException;
+
+    public void delete(String key) throws IOException;
+    
+    public void update(NutchDocument doc) throws IOException;
+
+    public void commit() throws IOException;
+    
+    public void close() throws IOException;
+
+    /** Returns a String describing the IndexWriter instance and the specific parameters it can take */
+	public String describe();
+}
+

Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexWriters.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.ExtensionPoint;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.util.ObjectCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Creates and caches {@link IndexWriter} implementing plugins. */
+public class IndexWriters {
+
+	public final static Logger LOG = LoggerFactory
+			.getLogger(IndexWriters.class);
+
+	private IndexWriter[] indexWriters;
+
+	public IndexWriters(Configuration conf) {
+		ObjectCache objectCache = ObjectCache.get(conf);
+		synchronized (objectCache) {
+			this.indexWriters = (IndexWriter[]) objectCache
+					.getObject(IndexWriter.class.getName());
+			if (this.indexWriters == null) {
+				try {
+					ExtensionPoint point = PluginRepository.get(conf)
+							.getExtensionPoint(IndexWriter.X_POINT_ID);
+					if (point == null)
+						throw new RuntimeException(IndexWriter.X_POINT_ID
+								+ " not found.");
+					Extension[] extensions = point.getExtensions();
+					HashMap<String, IndexWriter> indexerMap = new HashMap<String, IndexWriter>();
+					for (int i = 0; i < extensions.length; i++) {
+						Extension extension = extensions[i];
+						IndexWriter writer = (IndexWriter) extension
+								.getExtensionInstance();
+						LOG.info("Adding " + writer.getClass().getName());
+						if (!indexerMap.containsKey(writer.getClass().getName())) {
+							indexerMap.put(writer.getClass().getName(), writer);
+						}
+					}
+					objectCache.setObject(IndexWriter.class.getName(), indexerMap
+							.values().toArray(new IndexWriter[0]));
+				} catch (PluginRuntimeException e) {
+					throw new RuntimeException(e);
+				}
+				this.indexWriters = (IndexWriter[]) objectCache
+						.getObject(IndexWriter.class.getName());
+			}
+		}
+	}
+
+	public void open(JobConf job, String name) throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].open(job, name);
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	public void write(NutchDocument doc) throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].write(doc);
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	public void update(NutchDocument doc) throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].update(doc);
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	public void delete(String key) throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].delete(key);
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	public void close() throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].close();
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	public void commit() throws IOException {
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			try {
+				this.indexWriters[i].commit();
+			} catch (IOException ioe) {
+				throw ioe;
+			}
+		}
+	}
+
+	// lists the active IndexWriters and their configuration
+	public String describe() throws IOException {
+		StringBuffer buffer = new StringBuffer();
+		if (this.indexWriters.length == 0)
+			buffer.append("No IndexWriters activated - check your configuration\n");
+		else
+			buffer.append("Active IndexWriters :\n");
+		for (int i = 0; i < this.indexWriters.length; i++) {
+			buffer.append(this.indexWriters[i].describe()).append("\n");
+		}
+		return buffer.toString();
+	}
+
+}

Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerMapReduce.java Thu Mar  7 11:19:24 2013
@@ -56,6 +56,7 @@ implements Mapper<Text, Writable, Text, 
 
   public static final Logger LOG = LoggerFactory.getLogger(IndexerMapReduce.class);
 
+  public static final String INDEXER_PARAMS = "indexer.additional.params";
   public static final String INDEXER_DELETE = "indexer.delete";
   public static final String INDEXER_DELETE_ROBOTS_NOINDEX = "indexer.delete.robots.noindex";
   public static final String INDEXER_SKIP_NOTMODIFIED = "indexer.skip.notmodified";

Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java Thu Mar  7 11:19:24 2013
@@ -26,39 +26,32 @@ import org.apache.hadoop.mapred.RecordWr
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.Progressable;
 
-public class IndexerOutputFormat extends FileOutputFormat<Text, NutchIndexAction> {
+public class IndexerOutputFormat extends
+        FileOutputFormat<Text, NutchIndexAction> {
 
-  @Override
-  public RecordWriter<Text, NutchIndexAction> getRecordWriter(FileSystem ignored,
-      JobConf job, String name, Progressable progress) throws IOException {
-    
-    // populate JobConf with field indexing options
-    IndexingFilters filters = new IndexingFilters(job);
-    
-    final NutchIndexWriter[] writers =
-      NutchIndexWriterFactory.getNutchIndexWriters(job);
+    @Override
+    public RecordWriter<Text, NutchIndexAction> getRecordWriter(
+            FileSystem ignored, JobConf job, String name, Progressable progress)
+            throws IOException {
 
-    for (final NutchIndexWriter writer : writers) {
-      writer.open(job, name);
-    }
-    return new RecordWriter<Text, NutchIndexAction>() {
+        final IndexWriters writers = new IndexWriters(job);
+
+        writers.open(job, name);
+
+        return new RecordWriter<Text, NutchIndexAction>() {
 
-      public void close(Reporter reporter) throws IOException {
-        for (final NutchIndexWriter writer : writers) {
-          writer.close();
-        }
-      }
-
-      public void write(Text key, NutchIndexAction indexAction) throws IOException {
-        for (final NutchIndexWriter writer : writers) {
-          if (indexAction.action == NutchIndexAction.ADD) {
-            writer.write(indexAction.doc);
-          }
-          if (indexAction.action == NutchIndexAction.DELETE) {
-            writer.delete(key.toString());
-          }
-        }
-      }
-    };
-  }
+            public void close(Reporter reporter) throws IOException {
+                writers.close();
+            }
+
+            public void write(Text key, NutchIndexAction indexAction)
+                    throws IOException {
+                if (indexAction.action == NutchIndexAction.ADD) {
+                    writers.write(indexAction.doc);
+                } else if (indexAction.action == NutchIndexAction.DELETE) {
+                    writers.delete(key.toString());
+                }
+            }
+        };
+    }
 }

Added: nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexingJob.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.indexer.solr.SolrConstants;
+import org.apache.nutch.util.HadoopFSUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.TimingUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Generic indexer which relies on the plugins implementing IndexWriter
+ **/
+
+public class IndexingJob extends Configured implements Tool {
+
+    public static Logger LOG = LoggerFactory.getLogger(IndexingJob.class);
+
+    public IndexingJob() {
+        super(null);
+    }
+
+    public IndexingJob(Configuration conf) {
+        super(conf);
+    }
+
+    public void index(Path crawlDb, Path linkDb, List<Path> segments)
+            throws IOException {
+        // SOLR SPECIFIC COMMIT PARAM USED BY CRAWL CLASS
+        boolean noCommit = !getConf().getBoolean(SolrConstants.COMMIT_INDEX,
+                true);
+        index(crawlDb, linkDb, segments, noCommit, false, null);
+    }
+
+    public void index(Path crawlDb, Path linkDb, List<Path> segments,
+            boolean noCommit) throws IOException {
+        index(crawlDb, linkDb, segments, noCommit, false, null);
+    }
+
+    public void index(Path crawlDb, Path linkDb, List<Path> segments,
+            boolean noCommit, boolean deleteGone) throws IOException {
+        index(crawlDb, linkDb, segments, noCommit, deleteGone, null);
+    }
+
+    public void index(Path crawlDb, Path linkDb, List<Path> segments,
+            boolean noCommit, boolean deleteGone, String params)
+            throws IOException {
+        index(crawlDb, linkDb, segments, noCommit, deleteGone, params, false,
+                false);
+    }
+
+    public void index(Path crawlDb, Path linkDb, List<Path> segments,
+            boolean noCommit, boolean deleteGone, String params,
+            boolean filter, boolean normalize) throws IOException {
+
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        long start = System.currentTimeMillis();
+        LOG.info("Indexer: starting at " + sdf.format(start));
+
+        final JobConf job = new NutchJob(getConf());
+        job.setJobName("Indexer");
+
+        LOG.info("Indexer: deleting gone documents: " + deleteGone);
+        LOG.info("Indexer: URL filtering: " + filter);
+        LOG.info("Indexer: URL normalizing: " + normalize);   
+        
+        IndexWriters writers = new IndexWriters(getConf());
+        LOG.info(writers.describe());
+
+        IndexerMapReduce.initMRJob(crawlDb, linkDb, segments, job);
+
+        // NOW PASSED ON THE COMMAND LINE AS A HADOOP PARAM
+        // job.set(SolrConstants.SERVER_URL, solrUrl);
+
+        job.setBoolean(IndexerMapReduce.INDEXER_DELETE, deleteGone);
+        job.setBoolean(IndexerMapReduce.URL_FILTERING, filter);
+        job.setBoolean(IndexerMapReduce.URL_NORMALIZING, normalize);
+
+        if (params != null) {
+            job.set(IndexerMapReduce.INDEXER_PARAMS, params);
+        }
+
+        job.setReduceSpeculativeExecution(false);
+
+        final Path tmp = new Path("tmp_" + System.currentTimeMillis() + "-"
+                + new Random().nextInt());
+
+        FileOutputFormat.setOutputPath(job, tmp);
+        try {
+            JobClient.runJob(job);
+            // do the commits once and for all the reducers in one go
+            if (!noCommit) {
+                writers.open(job,"commit");
+                writers.commit();
+            }
+            long end = System.currentTimeMillis();
+            LOG.info("Indexer: finished at " + sdf.format(end) + ", elapsed: "
+                    + TimingUtil.elapsedTime(start, end));
+        } finally {
+            FileSystem.get(job).delete(tmp, true);
+        }
+    }
+
+    public int run(String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err
+                    .println("Usage: Indexer <crawldb> [-linkdb <linkdb>] [-params k1=v1&k2=v2...] (<segment> ... | -dir <segments>) [-noCommit] [-deleteGone] [-filter] [-normalize]");
+            IndexWriters writers = new IndexWriters(getConf());
+            System.err.println(writers.describe());
+            return -1;
+        }
+
+        final Path crawlDb = new Path(args[0]);
+        Path linkDb = null;
+
+        final List<Path> segments = new ArrayList<Path>();
+        String params = null;
+
+        boolean noCommit = false;
+        boolean deleteGone = false;
+        boolean filter = false;
+        boolean normalize = false;
+
+        for (int i = 1; i < args.length; i++) {
+            if (args[i].equals("-linkdb")) {
+                linkDb = new Path(args[++i]);
+            } else if (args[i].equals("-dir")) {
+                Path dir = new Path(args[++i]);
+                FileSystem fs = dir.getFileSystem(getConf());
+                FileStatus[] fstats = fs.listStatus(dir,
+                        HadoopFSUtil.getPassDirectoriesFilter(fs));
+                Path[] files = HadoopFSUtil.getPaths(fstats);
+                for (Path p : files) {
+                    segments.add(p);
+                }
+            } else if (args[i].equals("-noCommit")) {
+                noCommit = true;
+            } else if (args[i].equals("-deleteGone")) {
+                deleteGone = true;
+            } else if (args[i].equals("-filter")) {
+                filter = true;
+            } else if (args[i].equals("-normalize")) {
+                normalize = true;
+            } else if (args[i].equals("-params")) {
+                params = args[++i];
+            } else {
+                segments.add(new Path(args[i]));
+            }
+        }
+
+        try {
+            index(crawlDb, linkDb, segments, noCommit, deleteGone, params,
+                    filter, normalize);
+            return 0;
+        } catch (final Exception e) {
+            LOG.error("Indexer: " + StringUtils.stringifyException(e));
+            return -1;
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        final int res = ToolRunner.run(NutchConfiguration.create(),
+                new IndexingJob(), args);
+        System.exit(res);
+    }
+}

Modified: nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrUtils.java Thu Mar  7 11:19:24 2013
@@ -29,7 +29,7 @@ import java.net.MalformedURLException;
 
 public class SolrUtils {
 
-  public static Logger LOG = LoggerFactory.getLogger(SolrIndexer.class);
+  public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class);
 
   public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException {
     HttpClient client=new HttpClient();

Modified: nutch/trunk/src/plugin/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/build.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/plugin/build.xml (original)
+++ nutch/trunk/src/plugin/build.xml Thu Mar  7 11:19:24 2013
@@ -34,6 +34,7 @@
      <ant dir="index-more" target="deploy"/>
      <ant dir="index-static" target="deploy"/>
      <ant dir="index-metadata" target="deploy"/>
+     <ant dir="indexer-solr" target="deploy"/>
      <ant dir="language-identifier" target="deploy"/>
      <ant dir="lib-http" target="deploy"/>
      <ant dir="lib-nekohtml" target="deploy"/>
@@ -118,6 +119,7 @@
     <ant dir="index-more" target="clean"/>
     <ant dir="index-static" target="clean"/>
     <ant dir="index-metadata" target="clean"/>
+    <ant dir="indexer-solr" target="clean"/>
     <ant dir="language-identifier" target="clean"/>
     <ant dir="lib-commons-httpclient" target="clean"/>
     <ant dir="lib-http" target="clean"/>

Added: nutch/trunk/src/plugin/indexer-solr/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/build.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/build.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/build.xml Thu Mar  7 11:19:24 2013
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="indexer-solr" default="jar-core">
+
+  <import file="../build-plugin.xml" />
+
+</project>

Added: nutch/trunk/src/plugin/indexer-solr/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/ivy.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/ivy.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/ivy.xml Thu Mar  7 11:19:24 2013
@@ -0,0 +1,43 @@
+<?xml version="1.0" ?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<ivy-module version="1.0">
+  <info organisation="org.apache.nutch" module="${ant.project.name}">
+    <license name="Apache 2.0"/>
+    <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+    <description>
+        Apache Nutch
+    </description>
+  </info>
+
+  <configurations>
+    <include file="../../..//ivy/ivy-configurations.xml"/>
+  </configurations>
+
+  <publications>
+    <!--get the artifact from our module name-->
+    <artifact conf="master"/>
+  </publications>
+
+  <dependencies>
+   <dependency org="org.apache.solr" name="solr-solrj" rev="3.4.0"
+		conf="*->default"/>
+  </dependencies>
+  
+</ivy-module>

Added: nutch/trunk/src/plugin/indexer-solr/plugin.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/plugin.xml?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/plugin.xml (added)
+++ nutch/trunk/src/plugin/indexer-solr/plugin.xml Thu Mar  7 11:19:24 2013
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+  
+  http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<plugin id="indexer-solr" name="SOLRIndexWriter" version="1.0.0"
+  provider-name="nutch.apache.org">
+
+  <runtime>
+    <library name="indexer-solr.jar">
+      <export name="*" />
+    </library>
+
+     <library name="activation-1.1.jar"/>
+     <library name="commons-codec-1.4.jar"/>
+     <library name="commons-httpclient-3.1.jar"/>
+     <library name="commons-io-1.4.jar"/>
+     <library name="commons-logging-1.1.1.jar"/>
+     <library name="geronimo-stax-api_1.0_spec-1.0.1.jar"/>
+     <library name="jline-0.9.1.jar"/>
+     <library name="log4j-1.2.15.jar"/>
+     <library name="lucene-core-3.4.0.jar"/>
+     <library name="mail-1.4.1.jar"/>
+     <library name="slf4j-api-1.6.1.jar"/>
+     <library name="solr-solrj-3.4.0.jar"/>
+     <library name="stax-api-1.0.1.jar"/>
+     <library name="wstx-asl-3.2.7.jar"/>
+     <library name="zookeeper-3.3.1.jar"/>
+  </runtime>
+
+  <requires>
+    <import plugin="nutch-extensionpoints" />
+  </requires>
+
+  <extension id="org.apache.nutch.indexer.solr"
+    name="SOLR Index Writer"
+    point="org.apache.nutch.indexer.IndexWriter">
+    <implementation id="SOLRIndexWriter"
+      class="org.apache.nutch.indexwriter.solr.SolrIndexWriter" />
+  </extension>
+
+</plugin>

Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+public interface SolrConstants {
+  public static final String SOLR_PREFIX = "solr.";
+
+  public static final String SERVER_URL = SOLR_PREFIX + "server.url";
+
+  public static final String COMMIT_SIZE = SOLR_PREFIX + "commit.size";
+
+  public static final String MAPPING_FILE = SOLR_PREFIX + "mapping.file";
+
+  public static final String USE_AUTH = SOLR_PREFIX + "auth";
+
+  public static final String USERNAME = SOLR_PREFIX + "auth.username";
+
+  public static final String PASSWORD = SOLR_PREFIX + "auth.password";
+  
+  @Deprecated
+  public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index";
+  
+  @Deprecated
+  public static final String PARAMS = SOLR_PREFIX + "params";
+
+}

Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.nutch.indexer.IndexWriter;
+import org.apache.nutch.indexer.IndexerMapReduce;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.NutchField;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.DateUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrIndexWriter implements IndexWriter {
+
+    public static final Logger LOG = LoggerFactory
+            .getLogger(SolrIndexWriter.class);
+
+    private SolrServer solr;
+    private SolrMappingReader solrMapping;
+    private ModifiableSolrParams params;
+
+    private Configuration config;
+
+    private final List<SolrInputDocument> inputDocs = new ArrayList<SolrInputDocument>();
+
+    private int batchSize;
+    private int numDeletes = 0;
+    private boolean delete = false;
+
+    public void open(JobConf job, String name) throws IOException {
+        SolrServer server = SolrUtils.getCommonsHttpSolrServer(job);
+        init(server, job);
+    }
+
+    // package protected for tests
+    void init(SolrServer server, JobConf job) throws IOException {
+        solr = server;
+        batchSize = job.getInt(SolrConstants.COMMIT_SIZE, 1000);
+        solrMapping = SolrMappingReader.getInstance(job);
+        delete = job.getBoolean(IndexerMapReduce.INDEXER_DELETE, false);
+        // parse optional params
+        params = new ModifiableSolrParams();
+        String paramString = job.get(IndexerMapReduce.INDEXER_PARAMS);
+        if (paramString != null) {
+            String[] values = paramString.split("&");
+            for (String v : values) {
+                String[] kv = v.split("=");
+                if (kv.length < 2) {
+                    continue;
+                }
+                params.add(kv[0], kv[1]);
+            }
+        }
+    }
+
+    public void delete(String key) throws IOException {
+        if (delete) {
+            try {
+                solr.deleteById(key);
+                numDeletes++;
+            } catch (final SolrServerException e) {
+                throw makeIOException(e);
+            }
+        }
+    }
+
+    @Override
+    public void update(NutchDocument doc) throws IOException {
+        write(doc);
+    }
+
+    public void write(NutchDocument doc) throws IOException {
+        final SolrInputDocument inputDoc = new SolrInputDocument();
+        for (final Entry<String, NutchField> e : doc) {
+            for (final Object val : e.getValue().getValues()) {
+                // normalise the string representation for a Date
+                Object val2 = val;
+
+                if (val instanceof Date) {
+                    val2 = DateUtil.getThreadLocalDateFormat().format(val);
+                }
+
+                if (e.getKey().equals("content") || e.getKey().equals("title")) {
+                    val2 = SolrUtils.stripNonCharCodepoints((String) val);
+                }
+
+                inputDoc.addField(solrMapping.mapKey(e.getKey()), val2, e
+                        .getValue().getWeight());
+                String sCopy = solrMapping.mapCopyKey(e.getKey());
+                if (sCopy != e.getKey()) {
+                    inputDoc.addField(sCopy, val);
+                }
+            }
+        }
+
+        inputDoc.setDocumentBoost(doc.getWeight());
+        inputDocs.add(inputDoc);
+        if (inputDocs.size() + numDeletes >= batchSize) {
+            try {
+                LOG.info("Indexing " + Integer.toString(inputDocs.size())
+                        + " documents");
+                LOG.info("Deleting " + Integer.toString(numDeletes)
+                        + " documents");
+                numDeletes = 0;
+                UpdateRequest req = new UpdateRequest();
+                req.add(inputDocs);
+                req.setParams(params);
+                req.process(solr);
+            } catch (final SolrServerException e) {
+                throw makeIOException(e);
+            }
+            inputDocs.clear();
+        }
+    }
+
+    public void close() throws IOException {
+        try {
+            if (!inputDocs.isEmpty()) {
+                LOG.info("Indexing " + Integer.toString(inputDocs.size())
+                        + " documents");
+                if (numDeletes > 0) {
+                    LOG.info("Deleting " + Integer.toString(numDeletes)
+                            + " documents");
+                }
+                UpdateRequest req = new UpdateRequest();
+                req.add(inputDocs);
+                req.setParams(params);
+                req.process(solr);
+                inputDocs.clear();
+            }
+        } catch (final SolrServerException e) {
+            throw makeIOException(e);
+        }
+    }
+
+    @Override
+    public void commit() throws IOException {
+        try {
+            solr.commit();
+        } catch (SolrServerException e) {
+            throw makeIOException(e);
+        }
+    }
+
+    public static IOException makeIOException(SolrServerException e) {
+        final IOException ioe = new IOException();
+        ioe.initCause(e);
+        return ioe;
+    }
+
+    @Override
+    public Configuration getConf() {
+        return config;
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        config = conf;
+        String serverURL = conf.get(SolrConstants.SERVER_URL);
+        if (serverURL == null) {
+            String message = "Missing SOLR URL. Should be set via -D "
+                    + SolrConstants.SERVER_URL;
+            message+="\n"+describe();
+            LOG.error(message);
+            throw new RuntimeException(message);
+        }
+    }
+
+    public String describe(){
+    	StringBuffer sb = new StringBuffer("SOLRIndexWriter\n");
+    	sb.append("\t").append(SolrConstants.SERVER_URL).append(" : URL of the SOLR instance (mandatory)\n");
+    	sb.append("\t").append(SolrConstants.COMMIT_SIZE).append(" : buffer size when sending to SOLR (default 1000)\n");
+    	sb.append("\t").append(SolrConstants.MAPPING_FILE).append(" : name of the mapping file for fields (default solrindex-mapping.xml)\n");
+    	sb.append("\t").append(SolrConstants.USE_AUTH).append(" : use authentication (default false)\n");
+    	sb.append("\t").append(SolrConstants.USERNAME).append(" : use authentication (default false)\n");
+    	sb.append("\t").append(SolrConstants.USE_AUTH).append(" : username for authentication\n");
+    	sb.append("\t").append(SolrConstants.PASSWORD).append(" : password for authentication\n");
+    	return sb.toString();
+    }
+    
+}

Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrMappingReader.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.ObjectCache;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+public class SolrMappingReader {
+  public static Logger LOG = LoggerFactory.getLogger(SolrMappingReader.class);
+  
+  private Configuration conf;
+  
+  private Map<String, String> keyMap = new HashMap<String, String>();
+  private Map<String, String> copyMap = new HashMap<String, String>();
+  private String uniqueKey = "id";
+  
+  public static synchronized SolrMappingReader getInstance(Configuration conf) {
+    ObjectCache cache = ObjectCache.get(conf);
+    SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
+    if (instance == null) {
+      instance = new SolrMappingReader(conf);
+      cache.setObject(SolrMappingReader.class.getName(), instance);
+    }
+    return instance;
+  }
+
+  protected SolrMappingReader(Configuration conf) {
+    this.conf = conf;
+    parseMapping();
+  }
+
+  private void parseMapping() {    
+    InputStream ssInputStream = null;
+    ssInputStream = conf.getConfResourceAsInputStream(conf.get(SolrConstants.MAPPING_FILE, "solrindex-mapping.xml"));
+
+    InputSource inputSource = new InputSource(ssInputStream);
+    try {
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      Document document = builder.parse(inputSource);
+      Element rootElement = document.getDocumentElement();
+      NodeList fieldList = rootElement.getElementsByTagName("field");
+      if (fieldList.getLength() > 0) {
+        for (int i = 0; i < fieldList.getLength(); i++) {
+          Element element = (Element) fieldList.item(i);
+          LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+          keyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+        }
+      }
+      NodeList copyFieldList = rootElement.getElementsByTagName("copyField");
+      if (copyFieldList.getLength() > 0) {
+        for (int i = 0; i < copyFieldList.getLength(); i++) {
+          Element element = (Element) copyFieldList.item(i);
+          LOG.info("source: " + element.getAttribute("source") + " dest: " + element.getAttribute("dest"));
+          copyMap.put(element.getAttribute("source"), element.getAttribute("dest"));
+        }
+      }
+      NodeList uniqueKeyItem = rootElement.getElementsByTagName("uniqueKey");
+      if (uniqueKeyItem.getLength() > 1) {
+        LOG.warn("More than one unique key definitions found in solr index mapping, using default 'id'");
+        uniqueKey = "id";
+      }
+      else if (uniqueKeyItem.getLength() == 0) {
+        LOG.warn("No unique key definition found in solr index mapping using, default 'id'");
+      }
+      else{
+    	  uniqueKey = uniqueKeyItem.item(0).getFirstChild().getNodeValue();
+      }
+    } catch (MalformedURLException e) {
+        LOG.warn(e.toString());
+    } catch (SAXException e) {
+        LOG.warn(e.toString());
+    } catch (IOException e) {
+    	LOG.warn(e.toString());
+    } catch (ParserConfigurationException e) {
+    	LOG.warn(e.toString());
+    } 
+  }
+	  
+  public Map<String, String> getKeyMap() {
+    return keyMap;
+  }
+	  
+  public Map<String, String> getCopyMap() {
+    return copyMap;
+  }
+	  
+  public String getUniqueKey() {
+    return uniqueKey;
+  }
+
+  public String hasCopy(String key) {
+    if (copyMap.containsKey(key)) {
+      key = (String) copyMap.get(key);
+    }
+    return key;
+  }
+
+  public String mapKey(String key) throws IOException {
+    if(keyMap.containsKey(key)) {
+      key = (String) keyMap.get(key);
+    }
+    return key;
+  }
+
+  public String mapCopyKey(String key) throws IOException {
+    if(copyMap.containsKey(key)) {
+      key = (String) copyMap.get(key);
+    }
+    return key;
+  }
+}

Added: nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java?rev=1453776&view=auto
==============================================================================
--- nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (added)
+++ nutch/trunk/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java Thu Mar  7 11:19:24 2013
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexwriter.solr;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.params.HttpClientParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+
+import java.net.MalformedURLException;
+
+public class SolrUtils {
+
+  public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class);
+
+  public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException {
+    HttpClient client=new HttpClient();
+
+    // Check for username/password
+    if (job.getBoolean(SolrConstants.USE_AUTH, false)) {
+      String username = job.get(SolrConstants.USERNAME);
+
+      LOG.info("Authenticating as: " + username);
+
+      AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME);
+
+      client.getState().setCredentials(scope, new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD)));
+
+      HttpClientParams params = client.getParams();
+      params.setAuthenticationPreemptive(true);
+
+      client.setParams(params);
+    }
+
+    String serverURL = job.get(SolrConstants.SERVER_URL);
+    
+    return new CommonsHttpSolrServer(serverURL, client);
+  }
+
+  public static String stripNonCharCodepoints(String input) {
+    StringBuilder retval = new StringBuilder();
+    char ch;
+
+    for (int i = 0; i < input.length(); i++) {
+      ch = input.charAt(i);
+
+      // Strip all non-characters http://unicode.org/cldr/utility/list-unicodeset.jsp?a=[:Noncharacter_Code_Point=True:]
+      // and non-printable control characters except tabulator, new line and carriage return
+      if (ch % 0x10000 != 0xffff && // 0xffff - 0x10ffff range step 0x10000
+          ch % 0x10000 != 0xfffe && // 0xfffe - 0x10fffe range
+          (ch <= 0xfdd0 || ch >= 0xfdef) && // 0xfdd0 - 0xfdef
+          (ch > 0x1F || ch == 0x9 || ch == 0xa || ch == 0xd)) {
+
+        retval.append(ch);
+      }
+    }
+
+    return retval.toString();
+  }
+}

Modified: nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml?rev=1453776&r1=1453775&r2=1453776&view=diff
==============================================================================
--- nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml (original)
+++ nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml Thu Mar  7 11:19:24 2013
@@ -29,6 +29,10 @@
       name="Nutch Indexing Filter"/>
 
 <extension-point
+      id="org.apache.nutch.indexer.IndexWriter"
+      name="Nutch Index Writer"/>
+
+<extension-point
       id="org.apache.nutch.parse.Parser"
       name="Nutch Content Parser"/>