You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by do...@apache.org on 2010/10/26 13:17:53 UTC
svn commit: r1027471 - in /nutch/trunk: ./ conf/ ivy/
src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/fetcher/
src/java/org/apache/nutch/indexer/ src/java/org/apache/nutch/parse/
src/java/org/apache/nutch/storage/ src/java/org/apache/nutch/u...
Author: dogacan
Date: Tue Oct 26 11:17:52 2010
New Revision: 1027471
URL: http://svn.apache.org/viewvc?rev=1027471&view=rev
Log:
NUTCH-913 - Nutch should use new namespace for Gora
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/ivy/ivy.xml
nutch/trunk/ivy/ivysettings.xml
nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorMapper.java
nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorReducer.java
nutch/trunk/src/java/org/apache/nutch/crawl/InjectorJob.java
nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java
nutch/trunk/src/java/org/apache/nutch/fetcher/FetchEntry.java
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexerJob.java
nutch/trunk/src/java/org/apache/nutch/indexer/IndexerReducer.java
nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java
nutch/trunk/src/java/org/apache/nutch/storage/ParseStatus.java
nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatus.java
nutch/trunk/src/java/org/apache/nutch/storage/StorageUtils.java
nutch/trunk/src/java/org/apache/nutch/storage/WebPage.java
nutch/trunk/src/java/org/apache/nutch/storage/WebTableCreator.java
nutch/trunk/src/java/org/apache/nutch/util/IdentityPageReducer.java
nutch/trunk/src/java/org/apache/nutch/util/WebPageWritable.java
nutch/trunk/src/java/org/apache/nutch/util/domain/DomainStatistics.java
nutch/trunk/src/test/nutch-site.xml
nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java
nutch/trunk/src/test/org/apache/nutch/storage/TestGoraStorage.java
nutch/trunk/src/test/org/apache/nutch/util/AbstractNutchTest.java
nutch/trunk/src/test/org/apache/nutch/util/CrawlTestUtil.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Oct 26 11:17:52 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-913 Nutch should use new namespace for Gora (dogacan)
+
* NUTCH-714 Need a SFTP and SCP Protocol Handler (Sanjoy Ghosh, mattmann)
* NUTCH-894 Move statistical language identification from indexing to parsing step
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Tue Oct 26 11:17:52 2010
@@ -1037,7 +1037,7 @@
<property>
<name>storage.data.store.class</name>
- <value>org.gora.sql.store.SqlStore</value>
+ <value>org.apache.gora.sql.store.SqlStore</value>
<description>Default class for storing data</description>
</property>
Modified: nutch/trunk/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Tue Oct 26 11:17:52 2010
@@ -67,7 +67,7 @@
<exclude org="javax.jms"/>
</dependency>
- <dependency org="org.gora" name="gora-core" rev="0.1" conf="*->compile">
+ <dependency org="org.apache.gora" name="gora-core" rev="0.1" conf="*->compile">
<exclude org="com.sun.jdmk"/>
<exclude org="com.sun.jmx"/>
<exclude org="javax.jms"/>
@@ -110,7 +110,7 @@
<dependency org="org.hsqldb" name="hsqldb" rev="2.0.0" conf="*->default"/>
<dependency org="org.jdom" name="jdom" rev="1.1" conf="test->default"/>
- <dependency org="org.gora" name="gora-sql" rev="0.1" conf="test->compile">
+ <dependency org="org.apache.gora" name="gora-sql" rev="0.1" conf="test->compile">
<exclude org="com.sun.jdmk"/>
<exclude org="com.sun.jmx"/>
<exclude org="javax.jms"/>
Modified: nutch/trunk/ivy/ivysettings.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivysettings.xml?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/ivy/ivysettings.xml (original)
+++ nutch/trunk/ivy/ivysettings.xml Tue Oct 26 11:17:52 2010
@@ -75,6 +75,6 @@
rather than look for them online.
-->
<module organisation="org.apache.nutch" name=".*" resolver="internal"/>
- <module organisation="org.gora" name=".*" resolver="internal"/>
+ <module organisation="org.apache.gora" name=".*" resolver="internal"/>
</modules>
</ivysettings>
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateMapper.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateMapper.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateMapper.java Tue Oct 26 11:17:52 2010
@@ -15,7 +15,7 @@ import org.apache.nutch.scoring.ScoringF
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.WebPageWritable;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class DbUpdateMapper
extends GoraMapper<String, WebPage, String, NutchWritable> {
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateReducer.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdateReducer.java Tue Oct 26 11:17:52 2010
@@ -18,7 +18,7 @@ import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.WebPageWritable;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class DbUpdateReducer
extends GoraReducer<String, NutchWritable, String, WebPage> {
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdaterJob.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdaterJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/DbUpdaterJob.java Tue Oct 26 11:17:52 2010
@@ -17,7 +17,7 @@ import org.apache.nutch.storage.StorageU
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
-import org.gora.mapreduce.StringComparator;
+import org.apache.gora.mapreduce.StringComparator;
public class DbUpdaterJob extends Configured
implements Tool {
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorMapper.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorMapper.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorMapper.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorMapper.java Tue Oct 26 11:17:52 2010
@@ -12,7 +12,7 @@ import org.apache.nutch.scoring.ScoringF
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class GeneratorMapper
extends GoraMapper<String, WebPage, SelectorEntry, WebPage> {
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorReducer.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorReducer.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/GeneratorReducer.java Tue Oct 26 11:17:52 2010
@@ -12,7 +12,7 @@ import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
/** Reduce class for generate
*
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/InjectorJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/InjectorJob.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/InjectorJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/InjectorJob.java Tue Oct 26 11:17:52 2010
@@ -11,6 +11,9 @@ import java.util.TreeMap;
import org.apache.avro.util.Utf8;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraOutputFormat;
+import org.apache.gora.store.DataStore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
@@ -33,9 +36,6 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.GoraOutputFormat;
-import org.gora.store.DataStore;
/** This class takes a flat file of URLs and adds them to the of pages to be
* crawled. Useful for bootstrapping the system.
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java Tue Oct 26 11:17:52 2010
@@ -39,10 +39,10 @@ import org.apache.nutch.util.NutchConfig
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
/**
* Displays information about the entries of the webtable
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetchEntry.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetchEntry.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetchEntry.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetchEntry.java Tue Oct 26 11:17:52 2010
@@ -9,7 +9,7 @@ import org.apache.hadoop.conf.Configured
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.storage.WebPage;
-import org.gora.util.IOUtils;
+import org.apache.gora.util.IOUtils;
public class FetchEntry extends Configured implements Writable {
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java Tue Oct 26 11:17:52 2010
@@ -25,7 +25,7 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
/**
* Multi-threaded fetcher.
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java Tue Oct 26 11:17:52 2010
@@ -40,7 +40,7 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class FetcherReducer
extends GoraReducer<IntWritable, FetchEntry, String, WebPage> {
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerJob.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerJob.java Tue Oct 26 11:17:52 2010
@@ -22,8 +22,8 @@ import org.apache.nutch.storage.StorageU
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.StringComparator;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.StringComparator;
public abstract class IndexerJob
extends GoraMapper<String, WebPage, String, WebPage>
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexerReducer.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexerReducer.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexerReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexerReducer.java Tue Oct 26 11:17:52 2010
@@ -13,7 +13,7 @@ import org.apache.nutch.storage.StorageU
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.store.DataStore;
+import org.apache.gora.store.DataStore;
public class IndexerReducer
extends Reducer<String, WebPage, String, NutchDocument> {
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java Tue Oct 26 11:17:52 2010
@@ -23,7 +23,7 @@ import org.apache.nutch.util.IdentityPag
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class ParserJob extends GoraMapper<String, WebPage, String, WebPage>
implements Tool {
Modified: nutch/trunk/src/java/org/apache/nutch/storage/ParseStatus.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/storage/ParseStatus.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/storage/ParseStatus.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/storage/ParseStatus.java Tue Oct 26 11:17:52 2010
@@ -14,11 +14,11 @@ import org.apache.avro.specific.Specific
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class ParseStatus extends PersistentBase {
@@ -63,6 +63,7 @@ public class ParseStatus extends Persist
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:majorCode = (Integer)_value; break;
Modified: nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatus.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatus.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatus.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatus.java Tue Oct 26 11:17:52 2010
@@ -14,11 +14,11 @@ import org.apache.avro.specific.Specific
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class ProtocolStatus extends PersistentBase {
@@ -63,6 +63,7 @@ public class ProtocolStatus extends Pers
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:code = (Integer)_value; break;
Modified: nutch/trunk/src/java/org/apache/nutch/storage/StorageUtils.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/storage/StorageUtils.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/storage/StorageUtils.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/storage/StorageUtils.java Tue Oct 26 11:17:52 2010
@@ -4,17 +4,17 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraOutputFormat;
+import org.apache.gora.mapreduce.GoraReducer;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.nutch.metadata.Nutch;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.GoraOutputFormat;
-import org.gora.mapreduce.GoraReducer;
-import org.gora.persistency.Persistent;
-import org.gora.query.Query;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
public class StorageUtils {
@@ -48,7 +48,7 @@ public class StorageUtils {
getDataStoreClass(Configuration conf) throws ClassNotFoundException {
return (Class<? extends DataStore<K, V>>)
Class.forName(conf.get("storage.data.store.class",
- "org.gora.hbase.store.HBaseStore"));
+ "org.apache.gora.sql.store.SqlStore"));
}
public static <K, V> void initMapperJob(Job job,
Modified: nutch/trunk/src/java/org/apache/nutch/storage/WebPage.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/storage/WebPage.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/storage/WebPage.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/storage/WebPage.java Tue Oct 26 11:17:52 2010
@@ -14,11 +14,11 @@ import org.apache.avro.specific.Specific
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class WebPage extends PersistentBase {
@@ -124,6 +124,7 @@ public class WebPage extends PersistentB
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:baseUrl = (Utf8)_value; break;
Modified: nutch/trunk/src/java/org/apache/nutch/storage/WebTableCreator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/storage/WebTableCreator.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/storage/WebTableCreator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/storage/WebTableCreator.java Tue Oct 26 11:17:52 2010
@@ -1,7 +1,7 @@
package org.apache.nutch.storage;
import org.apache.nutch.util.NutchConfiguration;
-import org.gora.store.DataStore;
+import org.apache.gora.store.DataStore;
public class WebTableCreator {
public static void main(String[] args) throws Exception {
Modified: nutch/trunk/src/java/org/apache/nutch/util/IdentityPageReducer.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/IdentityPageReducer.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/IdentityPageReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/IdentityPageReducer.java Tue Oct 26 11:17:52 2010
@@ -3,7 +3,7 @@ package org.apache.nutch.util;
import java.io.IOException;
import org.apache.nutch.storage.WebPage;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class IdentityPageReducer
extends GoraReducer<String, WebPage, String, WebPage> {
Modified: nutch/trunk/src/java/org/apache/nutch/util/WebPageWritable.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/WebPageWritable.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/WebPageWritable.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/WebPageWritable.java Tue Oct 26 11:17:52 2010
@@ -8,7 +8,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.storage.WebPage;
-import org.gora.util.IOUtils;
+import org.apache.gora.util.IOUtils;
public class WebPageWritable extends Configured
implements Writable {
Modified: nutch/trunk/src/java/org/apache/nutch/util/domain/DomainStatistics.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/domain/DomainStatistics.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/domain/DomainStatistics.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/domain/DomainStatistics.java Tue Oct 26 11:17:52 2010
@@ -24,6 +24,9 @@ import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -50,9 +53,6 @@ import org.apache.nutch.util.NutchJobCon
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.TimingUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.query.Query;
-import org.gora.store.DataStore;
/**
* Extracts some very basic statistics about domains from the crawldb
Modified: nutch/trunk/src/test/nutch-site.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/nutch-site.xml?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/test/nutch-site.xml (original)
+++ nutch/trunk/src/test/nutch-site.xml Tue Oct 26 11:17:52 2010
@@ -18,7 +18,7 @@
<property>
<name>storage.data.store.class</name>
- <value>org.gora.sql.store.SqlStore</value>
+ <value>org.apache.gora.sql.store.SqlStore</value>
<description>Default class for storing data</description>
</property>
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java Tue Oct 26 11:17:52 2010
@@ -32,12 +32,12 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.AbstractNutchTest;
import org.apache.nutch.util.CrawlTestUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.sql.store.SqlStore;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
-import org.gora.util.ByteUtils;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.sql.store.SqlStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.util.ByteUtils;
import org.junit.Before;
/**
Modified: nutch/trunk/src/test/org/apache/nutch/storage/TestGoraStorage.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/storage/TestGoraStorage.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/storage/TestGoraStorage.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/storage/TestGoraStorage.java Tue Oct 26 11:17:52 2010
@@ -12,8 +12,8 @@ import java.util.concurrent.atomic.Atomi
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
import junit.framework.TestCase;
Modified: nutch/trunk/src/test/org/apache/nutch/util/AbstractNutchTest.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/util/AbstractNutchTest.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/util/AbstractNutchTest.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/util/AbstractNutchTest.java Tue Oct 26 11:17:52 2010
@@ -32,12 +32,12 @@ import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.StorageUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.sql.store.SqlStore;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
-import org.gora.util.ByteUtils;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.sql.store.SqlStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.util.ByteUtils;
/**
* This class provides common routines for setup/teardown of an in-memory data
Modified: nutch/trunk/src/test/org/apache/nutch/util/CrawlTestUtil.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/util/CrawlTestUtil.java?rev=1027471&r1=1027470&r2=1027471&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/util/CrawlTestUtil.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/util/CrawlTestUtil.java Tue Oct 26 11:17:52 2010
@@ -31,9 +31,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.nutch.crawl.URLWebPage;
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.handler.DefaultHandler;