You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by le...@apache.org on 2015/01/29 06:39:03 UTC
svn commit: r1655526 [24/26] - in /nutch/trunk: ./
src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/fetcher/
src/java/org/apache/nutch/indexer/ src/java/org/apache/nutch/metadata/
src/java/org/apache/nutch/net/ src/java/org/apache/nutch/net/p...
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java Thu Jan 29 05:38:59 2015
@@ -55,26 +55,26 @@ import org.slf4j.LoggerFactory;
* <li>configuration properties</li>
* <li>(additional) CrawlDatums of status linked (stemming from inlinks)</li>
* </ul>
- * </li>
- * </ul>
+ * </li> </ul>
*/
public class TestCrawlDbStates {
- private static final Logger LOG = LoggerFactory.getLogger(TestCrawlDbStates.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestCrawlDbStates.class);
protected static final byte[][] fetchDbStatusPairs = {
- { -1, STATUS_DB_UNFETCHED },
- { STATUS_FETCH_SUCCESS, STATUS_DB_FETCHED },
- { STATUS_FETCH_GONE, STATUS_DB_GONE },
- { STATUS_FETCH_REDIR_TEMP, STATUS_DB_REDIR_TEMP },
- { STATUS_FETCH_REDIR_PERM, STATUS_DB_REDIR_PERM },
+ { -1, STATUS_DB_UNFETCHED }, { STATUS_FETCH_SUCCESS, STATUS_DB_FETCHED },
+ { STATUS_FETCH_GONE, STATUS_DB_GONE },
+ { STATUS_FETCH_REDIR_TEMP, STATUS_DB_REDIR_TEMP },
+ { STATUS_FETCH_REDIR_PERM, STATUS_DB_REDIR_PERM },
{ STATUS_FETCH_NOTMODIFIED, STATUS_DB_NOTMODIFIED },
- { STATUS_FETCH_RETRY, -1 }, // fetch_retry does not have a CrawlDb counter-part
- { -1, STATUS_DB_DUPLICATE },
- };
+ { STATUS_FETCH_RETRY, -1 }, // fetch_retry does not have a CrawlDb
+ // counter-part
+ { -1, STATUS_DB_DUPLICATE }, };
/** tested {@link FetchSchedule} implementations */
- protected String[] schedules = {"DefaultFetchSchedule", "AdaptiveFetchSchedule"};
+ protected String[] schedules = { "DefaultFetchSchedule",
+ "AdaptiveFetchSchedule" };
/** CrawlDatum as result of a link */
protected final CrawlDatum linked = new CrawlDatum(STATUS_LINKED,
@@ -101,7 +101,7 @@ public class TestCrawlDbStates {
int retryMax = conf.getInt("db.fetch.retry.max", 3);
for (String sched : schedules) {
LOG.info("Testing state transitions with " + sched);
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl."+sched);
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched);
FetchSchedule schedule = FetchScheduleFactory
.getFetchSchedule(new JobConf(conf));
for (int i = 0; i < fetchDbStatusPairs.length; i++) {
@@ -138,8 +138,8 @@ public class TestCrawlDbStates {
}
String fromDbStatusName = (fromDbStatus == -1 ? "<not in CrawlDb>"
: getStatusName(fromDbStatus));
- String fetchStatusName = (fetchStatus == -1 ? "<only inlinks>" : CrawlDatum
- .getStatusName(fetchStatus));
+ String fetchStatusName = (fetchStatus == -1 ? "<only inlinks>"
+ : CrawlDatum.getStatusName(fetchStatus));
LOG.info(fromDbStatusName + " + " + fetchStatusName + " => "
+ getStatusName(toDbStatus));
List<CrawlDatum> values = new ArrayList<CrawlDatum>();
@@ -147,7 +147,8 @@ public class TestCrawlDbStates {
CrawlDatum fetch = null;
if (fetchStatus == -1) {
// nothing fetched, need at least one in-link
- if (l == 0) continue;
+ if (l == 0)
+ continue;
} else {
fetch = new CrawlDatum();
if (fromDb != null) {
@@ -183,7 +184,7 @@ public class TestCrawlDbStates {
}
}
}
- }
+ }
/**
* Test states after inject: inject must not modify the status of CrawlDatums
@@ -199,7 +200,7 @@ public class TestCrawlDbStates {
ScoringFilters scfilters = new ScoringFilters(conf);
for (String sched : schedules) {
LOG.info("Testing inject with " + sched);
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl."+sched);
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched);
FetchSchedule schedule = FetchScheduleFactory
.getFetchSchedule(new JobConf(conf));
List<CrawlDatum> values = new ArrayList<CrawlDatum>();
@@ -219,8 +220,8 @@ public class TestCrawlDbStates {
.getStatusName(fromDbStatus)) + " + "
+ getStatusName(STATUS_INJECTED) + " => "
+ getStatusName(toDbStatus));
- CrawlDatum injected = new CrawlDatum(STATUS_INJECTED,
- conf.getInt("db.fetch.interval.default", 2592000), 0.1f);
+ CrawlDatum injected = new CrawlDatum(STATUS_INJECTED, conf.getInt(
+ "db.fetch.interval.default", 2592000), 0.1f);
schedule.initializeSchedule(CrawlDbUpdateUtil.dummyURL, injected);
try {
scfilters.injectedScore(CrawlDbUpdateUtil.dummyURL, injected);
@@ -236,10 +237,10 @@ public class TestCrawlDbStates {
byte status = res.get(0).getStatus();
if (status != toDbStatus) {
fail("Inject for "
- + (fromDbStatus == -1 ? "" : getStatusName(fromDbStatus) + " and ")
- + getStatusName(STATUS_INJECTED)
- + " results in " + getStatusName(status)
- + " (expected: " + getStatusName(toDbStatus) + ")");
+ + (fromDbStatus == -1 ? "" : getStatusName(fromDbStatus)
+ + " and ") + getStatusName(STATUS_INJECTED) + " results in "
+ + getStatusName(status) + " (expected: "
+ + getStatusName(toDbStatus) + ")");
}
values.clear();
}
@@ -258,8 +259,7 @@ public class TestCrawlDbStates {
* <li>modified time is set</li>
* <li>re-fetch is triggered after a certain time to force the fetched content
* to be in a recent segment (old segments are deleted, see comments in
- * {@link CrawlDbReducer#reduce(Text, Iterator, OutputCollector, Reporter)}
- * </li>
+ * {@link CrawlDbReducer#reduce(Text, Iterator, OutputCollector, Reporter)}</li>
* </ul>
*/
@Test
@@ -270,7 +270,7 @@ public class TestCrawlDbStates {
for (String sched : schedules) {
String desc = "test notmodified by signature comparison + " + sched;
LOG.info(desc);
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl."+sched);
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched);
ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchNotModified(conf);
if (!crawlUtil.run(20)) {
fail("failed: " + desc);
@@ -280,8 +280,9 @@ public class TestCrawlDbStates {
for (String sched : schedules) {
String desc = "test notmodified by HTTP 304 + " + sched;
LOG.info(desc);
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl."+sched);
- ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchNotModifiedHttp304(conf);
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched);
+ ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchNotModifiedHttp304(
+ conf);
if (!crawlUtil.run(20)) {
fail("failed: " + desc);
}
@@ -294,7 +295,9 @@ public class TestCrawlDbStates {
protected long currFetchTime;
/** time the last fetch took place */
protected long lastFetchTime;
- /** time the document was fetched first (at all or after it has been changed) */
+ /**
+ * time the document was fetched first (at all or after it has been changed)
+ */
protected long firstFetchTime;
/** state in CrawlDb before the last fetch */
protected byte previousDbState;
@@ -304,18 +307,21 @@ public class TestCrawlDbStates {
private long maxFetchInterval;
private FetchSchedule schedule;
-
CrawlTestFetchNotModified(Configuration conf) {
super(conf);
- maxFetchInterval = conf.getLong("db.fetch.interval.max", 7776000); // default = 90 days
- maxFetchInterval += (24*60*60); // but take one day more to avoid false alarms
- maxFetchInterval *= 1000; // in milli-seconds
+ maxFetchInterval = conf.getLong("db.fetch.interval.max", 7776000); // default
+ // = 90
+ // days
+ maxFetchInterval += (24 * 60 * 60); // but take one day more to avoid
+ // false alarms
+ maxFetchInterval *= 1000; // in milli-seconds
schedule = FetchScheduleFactory.getFetchSchedule(new JobConf(conf));
}
@Override
protected boolean check(CrawlDatum result) {
- if (lastFetchTime > 0 && (currFetchTime - lastFetchTime) > maxFetchInterval) {
+ if (lastFetchTime > 0
+ && (currFetchTime - lastFetchTime) > maxFetchInterval) {
LOG.error("last effective fetch (HTTP 200, not HTTP 304), at "
+ new Date(lastFetchTime)
+ ", took place more than db.fetch.interval.max time, "
@@ -377,7 +383,6 @@ public class TestCrawlDbStates {
return false;
}
-
// test modified time
private boolean checkModifiedTime(CrawlDatum result, long modifiedTime) {
if (result.getModifiedTime() == 0) {
@@ -402,7 +407,7 @@ public class TestCrawlDbStates {
datum = super.fetch(datum, currentTime);
if (firstFetchTime == 0) {
firstFetchTime = currFetchTime;
- } else if ((currFetchTime - firstFetchTime) > (duration/2)) {
+ } else if ((currFetchTime - firstFetchTime) > (duration / 2)) {
// simulate a modification after "one year"
changeContent();
firstFetchTime = currFetchTime;
@@ -411,7 +416,8 @@ public class TestCrawlDbStates {
}
}
- protected class CrawlTestFetchNotModifiedHttp304 extends CrawlTestFetchNotModified {
+ protected class CrawlTestFetchNotModifiedHttp304 extends
+ CrawlTestFetchNotModified {
CrawlTestFetchNotModifiedHttp304(Configuration conf) {
super(conf);
@@ -424,13 +430,13 @@ public class TestCrawlDbStates {
previousDbState = datum.getStatus();
lastSignature = datum.getSignature();
int httpCode;
- /* document is "really" fetched (no HTTP 304)
- * - if last-modified time or signature are unset
- * (page has not been fetched before or fetch is forced)
- * - for test purposes, we simulate a modified after "one year"
+ /*
+ * document is "really" fetched (no HTTP 304) - if last-modified time or
+ * signature are unset (page has not been fetched before or fetch is
+ * forced) - for test purposes, we simulate a modified after "one year"
*/
if (datum.getModifiedTime() == 0 && datum.getSignature() == null
- || (currFetchTime - firstFetchTime) > (duration/2)) {
+ || (currFetchTime - firstFetchTime) > (duration / 2)) {
firstFetchTime = currFetchTime;
httpCode = 200;
datum.setStatus(STATUS_FETCH_SUCCESS);
@@ -450,8 +456,8 @@ public class TestCrawlDbStates {
/**
* NUTCH-1245: a fetch_gone should always result in a db_gone.
* <p>
- * Even in a long-running continuous crawl, when a gone page is
- * re-fetched several times over time.
+ * Even in a long-running continuous crawl, when a gone page is re-fetched
+ * several times over time.
* </p>
*/
@Test
@@ -476,8 +482,7 @@ public class TestCrawlDbStates {
LOG.info("NUTCH-1245 (misconfiguration): test with db.fetch.interval.default > (1.5 * db.fetch.interval.max)");
Configuration conf = CrawlDBTestUtil.createConfiguration();
int fetchIntervalMax = conf.getInt("db.fetch.interval.max", 0);
- conf.setInt("db.fetch.interval.default",
- 3 + (int) (fetchIntervalMax * 1.5));
+ conf.setInt("db.fetch.interval.default", 3 + (int) (fetchIntervalMax * 1.5));
ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestUtil(conf,
STATUS_FETCH_GONE, STATUS_DB_GONE);
if (!crawlUtil.run(0)) {
@@ -485,14 +490,12 @@ public class TestCrawlDbStates {
}
}
-
/**
- * Test whether signatures are reset for "content-less" states
- * (gone, redirect, etc.): otherwise, if this state is temporary
- * and the document appears again with the old content, it may
- * get marked as not_modified in CrawlDb just after the redirect
- * state. In this case we cannot expect content in segments.
- * Cf. NUTCH-1422: reset signature for redirects.
+ * Test whether signatures are reset for "content-less" states (gone,
+ * redirect, etc.): otherwise, if this state is temporary and the document
+ * appears again with the old content, it may get marked as not_modified in
+ * CrawlDb just after the redirect state. In this case we cannot expect
+ * content in segments. Cf. NUTCH-1422: reset signature for redirects.
*/
// TODO: can only test if solution is done in CrawlDbReducer
@Test
@@ -501,7 +504,7 @@ public class TestCrawlDbStates {
Configuration conf = CrawlDBTestUtil.createConfiguration();
for (String sched : schedules) {
LOG.info("Testing reset signature with " + sched);
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl."+sched);
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched);
ContinuousCrawlTestUtil crawlUtil = new CrawlTestSignatureReset(conf);
if (!crawlUtil.run(20)) {
fail("failed: signature not reset");
@@ -511,8 +514,7 @@ public class TestCrawlDbStates {
private class CrawlTestSignatureReset extends ContinuousCrawlTestUtil {
- byte[][] noContentStates = {
- { STATUS_FETCH_GONE, STATUS_DB_GONE },
+ byte[][] noContentStates = { { STATUS_FETCH_GONE, STATUS_DB_GONE },
{ STATUS_FETCH_REDIR_TEMP, STATUS_DB_REDIR_TEMP },
{ STATUS_FETCH_REDIR_PERM, STATUS_DB_REDIR_PERM } };
@@ -528,15 +530,15 @@ public class TestCrawlDbStates {
datum = super.fetch(datum, currentTime);
counter++;
// flip-flopping between successful fetch and one of content-less states
- if (counter%2 == 1) {
+ if (counter % 2 == 1) {
fetchState = STATUS_FETCH_SUCCESS;
} else {
- fetchState = noContentStates[(counter%6)/2][0];
+ fetchState = noContentStates[(counter % 6) / 2][0];
}
LOG.info("Step " + counter + ": fetched with "
+ getStatusName(fetchState));
datum.setStatus(fetchState);
- return datum;
+ return datum;
}
@Override
@@ -560,6 +562,4 @@ public class TestCrawlDbStates {
}
-
}
-
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java Thu Jan 29 05:38:59 2015
@@ -36,7 +36,7 @@ import org.junit.Test;
* Basic generator test. 1. Insert entries in crawldb 2. Generates entries to
* fetch 3. Verifies that number of generated urls match 4. Verifies that
* highest scoring urls are generated
- *
+ *
*/
public class TestGenerator {
@@ -71,7 +71,7 @@ public class TestGenerator {
/**
* Test that generator generates fetchlish ordered by score (desc).
- *
+ *
* @throws Exception
*/
@Test
@@ -82,8 +82,7 @@ public class TestGenerator {
ArrayList<URLCrawlDatum> list = new ArrayList<URLCrawlDatum>();
for (int i = 0; i <= 100; i++) {
- list.add(createURLCrawlDatum("http://aaa/" + pad(i),
- 1, i));
+ list.add(createURLCrawlDatum("http://aaa/" + pad(i), 1, i));
}
createCrawlDB(list);
@@ -94,7 +93,7 @@ public class TestGenerator {
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
ArrayList<URLCrawlDatum> l = readContents(fetchlist);
-
+
// sort urls by score desc
Collections.sort(l, new ScoreComparator());
@@ -132,18 +131,16 @@ public class TestGenerator {
/**
* Test that generator obeys the property "generate.max.per.host".
- * @throws Exception
+ *
+ * @throws Exception
*/
@Test
- public void testGenerateHostLimit() throws Exception{
+ public void testGenerateHostLimit() throws Exception {
ArrayList<URLCrawlDatum> list = new ArrayList<URLCrawlDatum>();
- list.add(createURLCrawlDatum("http://www.example.com/index1.html",
- 1, 1));
- list.add(createURLCrawlDatum("http://www.example.com/index2.html",
- 1, 1));
- list.add(createURLCrawlDatum("http://www.example.com/index3.html",
- 1, 1));
+ list.add(createURLCrawlDatum("http://www.example.com/index1.html", 1, 1));
+ list.add(createURLCrawlDatum("http://www.example.com/index2.html", 1, 1));
+ list.add(createURLCrawlDatum("http://www.example.com/index3.html", 1, 1));
createCrawlDB(list);
@@ -190,10 +187,11 @@ public class TestGenerator {
/**
* Test that generator obeys the property "generator.max.count" and
* "generator.count.per.domain".
- * @throws Exception
+ *
+ * @throws Exception
*/
@Test
- public void testGenerateDomainLimit() throws Exception{
+ public void testGenerateDomainLimit() throws Exception {
ArrayList<URLCrawlDatum> list = new ArrayList<URLCrawlDatum>();
list.add(createURLCrawlDatum("http://a.example.com/index.html", 1, 1));
@@ -204,7 +202,8 @@ public class TestGenerator {
Configuration myConfiguration = new Configuration(conf);
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 2);
- myConfiguration.set(Generator.GENERATOR_COUNT_MODE, Generator.GENERATOR_COUNT_VALUE_DOMAIN);
+ myConfiguration.set(Generator.GENERATOR_COUNT_MODE,
+ Generator.GENERATOR_COUNT_VALUE_DOMAIN);
Path generatedSegment = generateFetchlist(Integer.MAX_VALUE,
myConfiguration, false);
@@ -219,7 +218,8 @@ public class TestGenerator {
myConfiguration = new Configuration(myConfiguration);
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 3);
- generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
+ generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration,
+ false);
fetchlistPath = new Path(new Path(generatedSegment,
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
@@ -245,11 +245,12 @@ public class TestGenerator {
/**
* Test generator obeys the filter setting.
- * @throws Exception
- * @throws IOException
+ *
+ * @throws Exception
+ * @throws IOException
*/
@Test
- public void testFilter() throws IOException, Exception{
+ public void testFilter() throws IOException, Exception {
ArrayList<URLCrawlDatum> list = new ArrayList<URLCrawlDatum>();
@@ -267,7 +268,8 @@ public class TestGenerator {
Assert.assertNull("should be null (0 entries)", generatedSegment);
- generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
+ generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration,
+ false);
Path fetchlistPath = new Path(new Path(generatedSegment,
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
@@ -279,14 +281,16 @@ public class TestGenerator {
}
-
/**
* Read contents of fetchlist.
- * @param fetchlist path to Generated fetchlist
+ *
+ * @param fetchlist
+ * path to Generated fetchlist
* @return Generated {@link URLCrawlDatum} objects
* @throws IOException
*/
- private ArrayList<URLCrawlDatum> readContents(Path fetchlist) throws IOException {
+ private ArrayList<URLCrawlDatum> readContents(Path fetchlist)
+ throws IOException {
// verify results
SequenceFile.Reader reader = new SequenceFile.Reader(fs, fetchlist, conf);
@@ -307,8 +311,11 @@ public class TestGenerator {
/**
* Generate Fetchlist.
- * @param numResults number of results to generate
- * @param config Configuration to use
+ *
+ * @param numResults
+ * number of results to generate
+ * @param config
+ * Configuration to use
* @return path to generated segment
* @throws IOException
*/
@@ -318,14 +325,16 @@ public class TestGenerator {
Generator g = new Generator(config);
Path[] generatedSegment = g.generate(dbDir, segmentsDir, -1, numResults,
Long.MAX_VALUE, filter, false);
- if (generatedSegment==null) return null;
+ if (generatedSegment == null)
+ return null;
return generatedSegment[0];
}
/**
* Creates CrawlDB.
- *
- * @param list database contents
+ *
+ * @param list
+ * database contents
* @throws IOException
* @throws Exception
*/
@@ -342,9 +351,13 @@ public class TestGenerator {
/**
* Constructs new {@link URLCrawlDatum} from submitted parameters.
- * @param url url to use
- * @param fetchInterval {@link CrawlDatum#setFetchInterval(float)}
- * @param score {@link CrawlDatum#setScore(float)}
+ *
+ * @param url
+ * url to use
+ * @param fetchInterval
+ * {@link CrawlDatum#setFetchInterval(float)}
+ * @param score
+ * {@link CrawlDatum#setScore(float)}
* @return Constructed object
*/
private URLCrawlDatum createURLCrawlDatum(final String url,
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestInjector.java Thu Jan 29 05:38:59 2015
@@ -34,138 +34,143 @@ import org.junit.Before;
import org.junit.Test;
/**
- * Basic injector test:
- * 1. Creates a text file with urls
- * 2. Injects them into crawldb
- * 3. Reads crawldb entries and verifies contents
- * 4. Injects more urls into webdb
- * 5. Reads crawldb entries and verifies contents
+ * Basic injector test: 1. Creates a text file with urls 2. Injects them into
+ * crawldb 3. Reads crawldb entries and verifies contents 4. Injects more urls
+ * into webdb 5. Reads crawldb entries and verifies contents
*
*/
public class TestInjector {
private Configuration conf;
private FileSystem fs;
- final static Path testdir=new Path("build/test/inject-test");
+ final static Path testdir = new Path("build/test/inject-test");
Path crawldbPath;
Path urlPath;
-
+
@Before
public void setUp() throws Exception {
conf = CrawlDBTestUtil.createConfiguration();
- urlPath=new Path(testdir,"urls");
- crawldbPath=new Path(testdir,"crawldb");
- fs=FileSystem.get(conf);
- if (fs.exists(urlPath)) fs.delete(urlPath, false);
- if (fs.exists(crawldbPath)) fs.delete(crawldbPath, true);
+ urlPath = new Path(testdir, "urls");
+ crawldbPath = new Path(testdir, "crawldb");
+ fs = FileSystem.get(conf);
+ if (fs.exists(urlPath))
+ fs.delete(urlPath, false);
+ if (fs.exists(crawldbPath))
+ fs.delete(crawldbPath, true);
}
-
+
@After
- public void tearDown() throws IOException{
+ public void tearDown() throws IOException {
fs.delete(testdir, true);
}
@Test
public void testInject() throws IOException {
- ArrayList<String> urls=new ArrayList<String>();
- // We'll use a separate list for MD so we can still compare url with containsAll
- ArrayList<String> metadata=new ArrayList<String>();
- for(int i=0;i<100;i++) {
+ ArrayList<String> urls = new ArrayList<String>();
+ // We'll use a separate list for MD so we can still compare url with
+ // containsAll
+ ArrayList<String> metadata = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
urls.add("http://zzz.com/" + i + ".html");
- metadata.add("\tnutch.score=2." + i + "\tnutch.fetchInterval=171717\tkey=value");
+ metadata.add("\tnutch.score=2." + i
+ + "\tnutch.fetchInterval=171717\tkey=value");
}
CrawlDBTestUtil.generateSeedList(fs, urlPath, urls, metadata);
-
- Injector injector=new Injector(conf);
+
+ Injector injector = new Injector(conf);
injector.inject(crawldbPath, urlPath);
-
+
// verify results
- List<String>read=readCrawldb();
-
+ List<String> read = readCrawldb();
+
Collections.sort(read);
Collections.sort(urls);
Assert.assertEquals(urls.size(), read.size());
-
+
Assert.assertTrue(read.containsAll(urls));
Assert.assertTrue(urls.containsAll(read));
-
- //inject more urls
- ArrayList<String> urls2=new ArrayList<String>();
- for(int i=0;i<100;i++) {
+
+ // inject more urls
+ ArrayList<String> urls2 = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
urls2.add("http://xxx.com/" + i + ".html");
- // We'll overwrite previously injected records but preserve their original MD
+ // We'll overwrite previously injected records but preserve their original
+ // MD
urls2.add("http://zzz.com/" + i + ".html");
}
CrawlDBTestUtil.generateSeedList(fs, urlPath, urls2);
- injector=new Injector(conf);
+ injector = new Injector(conf);
conf.setBoolean("db.injector.update", true);
injector.inject(crawldbPath, urlPath);
urls.addAll(urls2);
-
+
// verify results
- read=readCrawldb();
+ read = readCrawldb();
Collections.sort(read);
Collections.sort(urls);
// We should have 100 less records because we've overwritten
Assert.assertEquals(urls.size() - 100, read.size());
-
+
Assert.assertTrue(read.containsAll(urls));
Assert.assertTrue(urls.containsAll(read));
-
+
// Check if we correctly preserved MD
Map<String, CrawlDatum> records = readCrawldbRecords();
-
+
// Iterate over the urls, we're looking for http://zzz.com/ prefixed URLs
// so we can check for MD and score and interval
Text writableKey = new Text("key");
Text writableValue = new Text("value");
for (String url : urls) {
- if (url.indexOf("http://zzz") == 0) {
+ if (url.indexOf("http://zzz") == 0) {
// Check for fetch interval
Assert.assertTrue(records.get(url).getFetchInterval() == 171717);
// Check for default score
Assert.assertTrue(records.get(url).getScore() != 1.0);
// Check for MD key=value
- Assert.assertEquals(writableValue, records.get(url).getMetaData().get(writableKey));
+ Assert.assertEquals(writableValue,
+ records.get(url).getMetaData().get(writableKey));
}
}
}
-
- private List<String> readCrawldb() throws IOException{
- Path dbfile=new Path(crawldbPath,CrawlDb.CURRENT_NAME + "/part-00000/data");
+
+ private List<String> readCrawldb() throws IOException {
+ Path dbfile = new Path(crawldbPath, CrawlDb.CURRENT_NAME
+ + "/part-00000/data");
System.out.println("reading:" + dbfile);
@SuppressWarnings("resource")
- SequenceFile.Reader reader=new SequenceFile.Reader(fs, dbfile, conf);
- ArrayList<String> read=new ArrayList<String>();
-
- READ:
- do {
- Text key=new Text();
- CrawlDatum value=new CrawlDatum();
- if(!reader.next(key, value)) break READ;
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, dbfile, conf);
+ ArrayList<String> read = new ArrayList<String>();
+
+ READ: do {
+ Text key = new Text();
+ CrawlDatum value = new CrawlDatum();
+ if (!reader.next(key, value))
+ break READ;
read.add(key.toString());
- } while(true);
+ } while (true);
return read;
}
-
- private HashMap<String,CrawlDatum> readCrawldbRecords() throws IOException{
- Path dbfile=new Path(crawldbPath,CrawlDb.CURRENT_NAME + "/part-00000/data");
+
+ private HashMap<String, CrawlDatum> readCrawldbRecords() throws IOException {
+ Path dbfile = new Path(crawldbPath, CrawlDb.CURRENT_NAME
+ + "/part-00000/data");
System.out.println("reading:" + dbfile);
@SuppressWarnings("resource")
- SequenceFile.Reader reader=new SequenceFile.Reader(fs, dbfile, conf);
- HashMap<String,CrawlDatum> read=new HashMap<String,CrawlDatum>();
-
- READ:
- do {
- Text key=new Text();
- CrawlDatum value=new CrawlDatum();
- if(!reader.next(key, value)) break READ;
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, dbfile, conf);
+ HashMap<String, CrawlDatum> read = new HashMap<String, CrawlDatum>();
+
+ READ: do {
+ Text key = new Text();
+ CrawlDatum value = new CrawlDatum();
+ if (!reader.next(key, value))
+ break READ;
read.put(key.toString(), value);
- } while(true);
+ } while (true);
return read;
}
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java Thu Jan 29 05:38:59 2015
@@ -35,41 +35,28 @@ import org.junit.Before;
import org.junit.Test;
public class TestLinkDbMerger {
- private static final Logger LOG = Logger.getLogger(TestLinkDbMerger.class.getName());
-
+ private static final Logger LOG = Logger.getLogger(TestLinkDbMerger.class
+ .getName());
+
String url10 = "http://example.com/foo";
- String[] urls10 = new String[] {
- "http://example.com/100",
- "http://example.com/101"
- };
+ String[] urls10 = new String[] { "http://example.com/100",
+ "http://example.com/101" };
String url11 = "http://example.com/";
- String[] urls11 = new String[] {
- "http://example.com/110",
- "http://example.com/111"
- };
-
+ String[] urls11 = new String[] { "http://example.com/110",
+ "http://example.com/111" };
+
String url20 = "http://example.com/";
- String[] urls20 = new String[] {
- "http://foo.com/200",
- "http://foo.com/201"
- };
+ String[] urls20 = new String[] { "http://foo.com/200", "http://foo.com/201" };
String url21 = "http://example.com/bar";
- String[] urls21 = new String[] {
- "http://foo.com/210",
- "http://foo.com/211"
- };
-
+ String[] urls21 = new String[] { "http://foo.com/210", "http://foo.com/211" };
+
String[] urls10_expected = urls10;
- String[] urls11_expected = new String[] {
- urls11[0],
- urls11[1],
- urls20[0],
- urls20[1]
- };
+ String[] urls11_expected = new String[] { urls11[0], urls11[1], urls20[0],
+ urls20[1] };
String[] urls20_expected = urls11_expected;
String[] urls21_expected = urls21;
-
+
TreeMap<String, String[]> init1 = new TreeMap<String, String[]>();
TreeMap<String, String[]> init2 = new TreeMap<String, String[]>();
HashMap<String, String[]> expected = new HashMap<String, String[]>();
@@ -77,7 +64,7 @@ public class TestLinkDbMerger {
Path testDir;
FileSystem fs;
LinkDbReader reader;
-
+
@Before
public void setUp() throws Exception {
init1.put(url10, urls10);
@@ -90,20 +77,22 @@ public class TestLinkDbMerger {
expected.put(url21, urls21_expected);
conf = NutchConfiguration.create();
fs = FileSystem.get(conf);
- testDir = new Path("build/test/test-linkdb-" +
- new java.util.Random().nextInt());
+ testDir = new Path("build/test/test-linkdb-"
+ + new java.util.Random().nextInt());
fs.mkdirs(testDir);
}
-
+
@After
public void tearDown() {
try {
if (fs.exists(testDir))
fs.delete(testDir, true);
- } catch (Exception e) { }
+ } catch (Exception e) {
+ }
try {
reader.close();
- } catch (Exception e) { }
+ } catch (Exception e) {
+ }
}
@Test
@@ -118,7 +107,7 @@ public class TestLinkDbMerger {
createLinkDb(conf, fs, linkdb2, init2);
LinkDbMerger merger = new LinkDbMerger(conf);
LOG.fine("* merging linkdbs to " + output);
- merger.merge(output, new Path[]{linkdb1, linkdb2}, false, false);
+ merger.merge(output, new Path[] { linkdb1, linkdb2 }, false, false);
LOG.fine("* reading linkdb: " + output);
reader = new LinkDbReader(conf, output);
Iterator<String> it = expected.keySet().iterator();
@@ -132,7 +121,7 @@ public class TestLinkDbMerger {
ArrayList<String> links = new ArrayList<String>();
Iterator<?> it2 = inlinks.iterator();
while (it2.hasNext()) {
- Inlink in = (Inlink)it2.next();
+ Inlink in = (Inlink) it2.next();
links.add(in.getFromUrl());
}
for (int i = 0; i < vals.length; i++) {
@@ -143,11 +132,13 @@ public class TestLinkDbMerger {
reader.close();
fs.delete(testDir, true);
}
-
- private void createLinkDb(Configuration config, FileSystem fs, Path linkdb, TreeMap<String, String[]> init) throws Exception {
+
+ private void createLinkDb(Configuration config, FileSystem fs, Path linkdb,
+ TreeMap<String, String[]> init) throws Exception {
LOG.fine("* creating linkdb: " + linkdb);
Path dir = new Path(linkdb, LinkDb.CURRENT_NAME);
- MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class);
+ MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir,
+ "part-00000").toString(), Text.class, Inlinks.class);
Iterator<String> it = init.keySet().iterator();
while (it.hasNext()) {
String key = it.next();
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestSignatureFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestSignatureFactory.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestSignatureFactory.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestSignatureFactory.java Thu Jan 29 05:38:59 2015
@@ -25,9 +25,9 @@ public class TestSignatureFactory {
@Test
public void testGetSignature() {
- Configuration conf=NutchConfiguration.create();
- Signature signature1=SignatureFactory.getSignature(conf);
- Signature signature2=SignatureFactory.getSignature(conf);
+ Configuration conf = NutchConfiguration.create();
+ Signature signature1 = SignatureFactory.getSignature(conf);
+ Signature signature2 = SignatureFactory.getSignature(conf);
Assert.assertNotNull(signature1);
Assert.assertNotNull(signature2);
Assert.assertEquals(signature1, signature2);
Modified: nutch/trunk/src/test/org/apache/nutch/fetcher/TestFetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/fetcher/TestFetcher.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/fetcher/TestFetcher.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/fetcher/TestFetcher.java Thu Jan 29 05:38:59 2015
@@ -39,17 +39,13 @@ import org.junit.Test;
import org.mortbay.jetty.Server;
/**
- * Basic fetcher test
- * 1. generate seedlist
- * 2. inject
- * 3. generate
- * 3. fetch
- * 4. Verify contents
- *
+ * Basic fetcher test 1. generate seedlist 2. inject 3. generate 3. fetch 4.
+ * Verify contents
+ *
*/
public class TestFetcher {
- final static Path testdir=new Path("build/test/fetch-test");
+ final static Path testdir = new Path("build/test/fetch-test");
Configuration conf;
FileSystem fs;
Path crawldbPath;
@@ -58,112 +54,118 @@ public class TestFetcher {
Server server;
@Before
- public void setUp() throws Exception{
- conf=CrawlDBTestUtil.createConfiguration();
- fs=FileSystem.get(conf);
+ public void setUp() throws Exception {
+ conf = CrawlDBTestUtil.createConfiguration();
+ fs = FileSystem.get(conf);
fs.delete(testdir, true);
- urlPath=new Path(testdir,"urls");
- crawldbPath=new Path(testdir,"crawldb");
- segmentsPath=new Path(testdir,"segments");
- server=CrawlDBTestUtil.getServer(conf.getInt("content.server.port",50000), "build/test/data/fetch-test-site");
+ urlPath = new Path(testdir, "urls");
+ crawldbPath = new Path(testdir, "crawldb");
+ segmentsPath = new Path(testdir, "segments");
+ server = CrawlDBTestUtil.getServer(
+ conf.getInt("content.server.port", 50000),
+ "build/test/data/fetch-test-site");
server.start();
}
@After
- public void tearDown() throws Exception{
+ public void tearDown() throws Exception {
server.stop();
fs.delete(testdir, true);
}
-
+
@Test
public void testFetch() throws IOException {
-
- //generate seedlist
- ArrayList<String> urls=new ArrayList<String>();
-
- addUrl(urls,"index.html");
- addUrl(urls,"pagea.html");
- addUrl(urls,"pageb.html");
- addUrl(urls,"dup_of_pagea.html");
- addUrl(urls,"nested_spider_trap.html");
- addUrl(urls,"exception.html");
-
+
+ // generate seedlist
+ ArrayList<String> urls = new ArrayList<String>();
+
+ addUrl(urls, "index.html");
+ addUrl(urls, "pagea.html");
+ addUrl(urls, "pageb.html");
+ addUrl(urls, "dup_of_pagea.html");
+ addUrl(urls, "nested_spider_trap.html");
+ addUrl(urls, "exception.html");
+
CrawlDBTestUtil.generateSeedList(fs, urlPath, urls);
-
- //inject
- Injector injector=new Injector(conf);
+
+ // inject
+ Injector injector = new Injector(conf);
injector.inject(crawldbPath, urlPath);
- //generate
- Generator g=new Generator(conf);
+ // generate
+ Generator g = new Generator(conf);
Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
Long.MAX_VALUE, Long.MAX_VALUE, false, false);
- long time=System.currentTimeMillis();
- //fetch
- Fetcher fetcher=new Fetcher(conf);
+ long time = System.currentTimeMillis();
+ // fetch
+ Fetcher fetcher = new Fetcher(conf);
// Set fetcher.parse to true
conf.setBoolean("fetcher.parse", true);
fetcher.fetch(generatedSegment[0], 1);
- time=System.currentTimeMillis()-time;
-
- //verify politeness, time taken should be more than (num_of_pages +1)*delay
- int minimumTime=(int) ((urls.size()+1)*1000*conf.getFloat("fetcher.server.delay",5));
+ time = System.currentTimeMillis() - time;
+
+ // verify politeness, time taken should be more than (num_of_pages +1)*delay
+ int minimumTime = (int) ((urls.size() + 1) * 1000 * conf.getFloat(
+ "fetcher.server.delay", 5));
Assert.assertTrue(time > minimumTime);
-
- //verify content
- Path content=new Path(new Path(generatedSegment[0], Content.DIR_NAME),"part-00000/data");
+
+ // verify content
+ Path content = new Path(new Path(generatedSegment[0], Content.DIR_NAME),
+ "part-00000/data");
@SuppressWarnings("resource")
- SequenceFile.Reader reader=new SequenceFile.Reader(fs, content, conf);
-
- ArrayList<String> handledurls=new ArrayList<String>();
-
- READ_CONTENT:
- do {
- Text key=new Text();
- Content value=new Content();
- if(!reader.next(key, value)) break READ_CONTENT;
- String contentString=new String(value.getContent());
- if(contentString.indexOf("Nutch fetcher test page")!=-1) {
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, content, conf);
+
+ ArrayList<String> handledurls = new ArrayList<String>();
+
+ READ_CONTENT: do {
+ Text key = new Text();
+ Content value = new Content();
+ if (!reader.next(key, value))
+ break READ_CONTENT;
+ String contentString = new String(value.getContent());
+ if (contentString.indexOf("Nutch fetcher test page") != -1) {
handledurls.add(key.toString());
}
- } while(true);
+ } while (true);
reader.close();
Collections.sort(urls);
Collections.sort(handledurls);
- //verify that enough pages were handled
+ // verify that enough pages were handled
Assert.assertEquals(urls.size(), handledurls.size());
- //verify that correct pages were handled
+ // verify that correct pages were handled
Assert.assertTrue(handledurls.containsAll(urls));
Assert.assertTrue(urls.containsAll(handledurls));
-
+
handledurls.clear();
- //verify parse data
- Path parseData = new Path(new Path(generatedSegment[0], ParseData.DIR_NAME),"part-00000/data");
+ // verify parse data
+ Path parseData = new Path(
+ new Path(generatedSegment[0], ParseData.DIR_NAME), "part-00000/data");
reader = new SequenceFile.Reader(fs, parseData, conf);
-
- READ_PARSE_DATA:
- do {
+
+ READ_PARSE_DATA: do {
Text key = new Text();
ParseData value = new ParseData();
- if(!reader.next(key, value)) break READ_PARSE_DATA;
- // make sure they all contain "nutch.segment.name" and "nutch.content.digest"
+ if (!reader.next(key, value))
+ break READ_PARSE_DATA;
+ // make sure they all contain "nutch.segment.name" and
+ // "nutch.content.digest"
// keys in parse metadata
Metadata contentMeta = value.getContentMeta();
- if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null
- && contentMeta.get(Nutch.SIGNATURE_KEY) != null) {
+ if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null
+ && contentMeta.get(Nutch.SIGNATURE_KEY) != null) {
handledurls.add(key.toString());
}
- } while(true);
-
+ } while (true);
+
Collections.sort(handledurls);
Assert.assertEquals(urls.size(), handledurls.size());
@@ -173,9 +175,10 @@ public class TestFetcher {
}
private void addUrl(ArrayList<String> urls, String page) {
- urls.add("http://127.0.0.1:" + server.getConnectors()[0].getPort() + "/" + page);
+ urls.add("http://127.0.0.1:" + server.getConnectors()[0].getPort() + "/"
+ + page);
}
-
+
@Test
public void testAgentNameCheck() {
Modified: nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexingFilters.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexingFilters.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexingFilters.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/indexer/TestIndexingFilters.java Thu Jan 29 05:38:59 2015
@@ -33,13 +33,14 @@ public class TestIndexingFilters {
/**
* Test behaviour when defined filter does not exist.
+ *
* @throws IndexingException
*/
@Test
public void testNonExistingIndexingFilter() throws IndexingException {
Configuration conf = NutchConfiguration.create();
- conf.addResource("nutch-default.xml");
- conf.addResource("crawl-tests.xml");
+ conf.addResource("nutch-default.xml");
+ conf.addResource("crawl-tests.xml");
String class1 = "NonExistingFilter";
String class2 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
@@ -47,34 +48,35 @@ public class TestIndexingFilters {
IndexingFilters filters = new IndexingFilters(conf);
filters.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
- new ParseStatus(), "title", new Outlink[0], new Metadata())), new Text(
- "http://www.example.com/"), new CrawlDatum(), new Inlinks());
+ new ParseStatus(), "title", new Outlink[0], new Metadata())), new Text(
+ "http://www.example.com/"), new CrawlDatum(), new Inlinks());
}
/**
* Test behaviour when NutchDOcument is null
*/
@Test
- public void testNutchDocumentNullIndexingFilter() throws IndexingException{
+ public void testNutchDocumentNullIndexingFilter() throws IndexingException {
Configuration conf = NutchConfiguration.create();
conf.addResource("nutch-default.xml");
conf.addResource("crawl-tests.xml");
IndexingFilters filters = new IndexingFilters(conf);
- NutchDocument doc = filters.filter(null, new ParseImpl("text", new ParseData(
- new ParseStatus(), "title", new Outlink[0], new Metadata())), new Text(
- "http://www.example.com/"), new CrawlDatum(), new Inlinks());
-
+ NutchDocument doc = filters.filter(null, new ParseImpl("text",
+ new ParseData(new ParseStatus(), "title", new Outlink[0],
+ new Metadata())), new Text("http://www.example.com/"),
+ new CrawlDatum(), new Inlinks());
+
Assert.assertNull(doc);
}
/**
* Test behaviour when reset the index filter order will not take effect
- *
+ *
* @throws IndexingException
*/
@Test
- public void testFilterCacheIndexingFilter() throws IndexingException{
+ public void testFilterCacheIndexingFilter() throws IndexingException {
Configuration conf = NutchConfiguration.create();
conf.addResource("nutch-default.xml");
conf.addResource("crawl-tests.xml");
@@ -83,24 +85,26 @@ public class TestIndexingFilters {
conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);
IndexingFilters filters1 = new IndexingFilters(conf);
- NutchDocument fdoc1 = filters1.filter(new NutchDocument(),new ParseImpl("text",new ParseData(
- new ParseStatus(),"title",new Outlink[0],new Metadata())),new Text("http://www.example.com/"),
- new CrawlDatum(),new Inlinks());
+ NutchDocument fdoc1 = filters1.filter(new NutchDocument(), new ParseImpl(
+ "text", new ParseData(new ParseStatus(), "title", new Outlink[0],
+ new Metadata())), new Text("http://www.example.com/"),
+ new CrawlDatum(), new Inlinks());
// add another index filter
String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
// set content metadata
Metadata md = new Metadata();
- md.add("example","data");
+ md.add("example", "data");
// set content metadata property defined in MetadataIndexer
- conf.set("index.content.md","example");
+ conf.set("index.content.md", "example");
// add MetadataIndxer filter
conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
IndexingFilters filters2 = new IndexingFilters(conf);
- NutchDocument fdoc2 = filters2.filter(new NutchDocument(),new ParseImpl("text",new ParseData(
- new ParseStatus(),"title",new Outlink[0],md)),new Text("http://www.example.com/"),
- new CrawlDatum(),new Inlinks());
- Assert.assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size());
+ NutchDocument fdoc2 = filters2.filter(new NutchDocument(), new ParseImpl(
+ "text", new ParseData(new ParseStatus(), "title", new Outlink[0], md)),
+ new Text("http://www.example.com/"), new CrawlDatum(), new Inlinks());
+ Assert.assertEquals(fdoc1.getFieldNames().size(), fdoc2.getFieldNames()
+ .size());
}
}
Modified: nutch/trunk/src/test/org/apache/nutch/metadata/TestMetadata.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/metadata/TestMetadata.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/metadata/TestMetadata.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/metadata/TestMetadata.java Thu Jan 29 05:38:59 2015
@@ -33,7 +33,6 @@ public class TestMetadata {
private static final String CONTENTTYPE = "contenttype";
-
/**
* Test to ensure that only non-null values get written when the
* {@link Metadata} object is written using a Writeable.
@@ -280,4 +279,3 @@ public class TestMetadata {
}
}
-
Modified: nutch/trunk/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java Thu Jan 29 05:38:59 2015
@@ -29,7 +29,7 @@ import org.junit.Test;
/**
* JUnit based tests of class
* {@link org.apache.nutch.metadata.SpellCheckedMetadata}.
- *
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
@@ -40,20 +40,20 @@ public class TestSpellCheckedMetadata {
/** Test for the <code>getNormalizedName(String)</code> method. */
@Test
public void testGetNormalizedName() {
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("Content-Type"));
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("ContentType"));
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("Content-type"));
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("contenttype"));
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("contentype"));
- Assert.assertEquals("Content-Type", SpellCheckedMetadata
- .getNormalizedName("contntype"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("Content-Type"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("ContentType"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("Content-type"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("contenttype"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("contentype"));
+ Assert.assertEquals("Content-Type",
+ SpellCheckedMetadata.getNormalizedName("contntype"));
}
-
+
/** Test for the <code>add(String, String)</code> method. */
@Test
public void testAdd() {
@@ -253,8 +253,8 @@ public class TestSpellCheckedMetadata {
}
/**
- * IO Test method, usable only when you plan to do changes in metadata
- * to measure relative performance impact.
+ * IO Test method, usable only when you plan to do changes in metadata to
+ * measure relative performance impact.
*/
@Test
public final void testHandlingSpeed() {
Modified: nutch/trunk/src/test/org/apache/nutch/net/TestURLFilters.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/net/TestURLFilters.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/net/TestURLFilters.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/net/TestURLFilters.java Thu Jan 29 05:38:59 2015
@@ -24,6 +24,7 @@ public class TestURLFilters {
/**
* Testcase for NUTCH-325.
+ *
* @throws URLFilterException
*/
@Test
Modified: nutch/trunk/src/test/org/apache/nutch/net/TestURLNormalizers.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/net/TestURLNormalizers.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/net/TestURLNormalizers.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/net/TestURLNormalizers.java Thu Jan 29 05:38:59 2015
@@ -31,41 +31,53 @@ public class TestURLNormalizers {
String clazz1 = "org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer";
String clazz2 = "org.apache.nutch.net.urlnormalizer.basic.BasicURLNormalizer";
conf.set("urlnormalizer.order", clazz1 + " " + clazz2);
-
- URLNormalizers normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
-
+
+ URLNormalizers normalizers = new URLNormalizers(conf,
+ URLNormalizers.SCOPE_DEFAULT);
+
Assert.assertNotNull(normalizers);
try {
- normalizers.normalize("http://www.example.com/", URLNormalizers.SCOPE_DEFAULT);
+ normalizers.normalize("http://www.example.com/",
+ URLNormalizers.SCOPE_DEFAULT);
} catch (MalformedURLException mue) {
Assert.fail(mue.toString());
}
// NUTCH-1011 - Get rid of superfluous slashes
try {
- String normalizedSlashes = normalizers.normalize("http://www.example.com//path/to//somewhere.html", URLNormalizers.SCOPE_DEFAULT);
- Assert.assertEquals(normalizedSlashes, "http://www.example.com/path/to/somewhere.html");
+ String normalizedSlashes = normalizers.normalize(
+ "http://www.example.com//path/to//somewhere.html",
+ URLNormalizers.SCOPE_DEFAULT);
+ Assert.assertEquals(normalizedSlashes,
+ "http://www.example.com/path/to/somewhere.html");
} catch (MalformedURLException mue) {
Assert.fail(mue.toString());
}
-
+
// HostNormalizer NUTCH-1319
try {
- String normalizedHost = normalizers.normalize("http://www.example.org//path/to//somewhere.html", URLNormalizers.SCOPE_DEFAULT);
- Assert.assertEquals(normalizedHost, "http://example.org/path/to/somewhere.html");
+ String normalizedHost = normalizers.normalize(
+ "http://www.example.org//path/to//somewhere.html",
+ URLNormalizers.SCOPE_DEFAULT);
+ Assert.assertEquals(normalizedHost,
+ "http://example.org/path/to/somewhere.html");
} catch (MalformedURLException mue) {
Assert.fail(mue.toString());
}
-
+
// check the order
int pos1 = -1, pos2 = -1;
- URLNormalizer[] impls = normalizers.getURLNormalizers(URLNormalizers.SCOPE_DEFAULT);
+ URLNormalizer[] impls = normalizers
+ .getURLNormalizers(URLNormalizers.SCOPE_DEFAULT);
for (int i = 0; i < impls.length; i++) {
- if (impls[i].getClass().getName().equals(clazz1)) pos1 = i;
- if (impls[i].getClass().getName().equals(clazz2)) pos2 = i;
+ if (impls[i].getClass().getName().equals(clazz1))
+ pos1 = i;
+ if (impls[i].getClass().getName().equals(clazz2))
+ pos2 = i;
}
if (pos1 != -1 && pos2 != -1) {
- Assert.assertTrue("RegexURLNormalizer before BasicURLNormalizer", pos1 < pos2);
+ Assert.assertTrue("RegexURLNormalizer before BasicURLNormalizer",
+ pos1 < pos2);
}
}
}
Modified: nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java Thu Jan 29 05:38:59 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.nutch.parse;
import org.apache.nutch.parse.Outlink;
@@ -34,54 +34,66 @@ import org.junit.Test;
public class TestOutlinkExtractor {
private static Configuration conf = NutchConfiguration.create();
-
+
@Test
public void testGetNoOutlinks() {
- Outlink[] outlinks = null;
-
+ Outlink[] outlinks = null;
+
outlinks = OutlinkExtractor.getOutlinks(null, conf);
Assert.assertNotNull(outlinks);
Assert.assertEquals(0, outlinks.length);
-
+
outlinks = OutlinkExtractor.getOutlinks("", conf);
Assert.assertNotNull(outlinks);
Assert.assertEquals(0, outlinks.length);
}
-
+
@Test
public void testGetOutlinksHttp() {
- Outlink[] outlinks = OutlinkExtractor.getOutlinks(
- "Test with http://www.nutch.org/index.html is it found? " +
- "What about www.google.com at http://www.google.de " +
- "A longer URL could be http://www.sybit.com/solutions/portals.html", conf);
-
+ Outlink[] outlinks = OutlinkExtractor
+ .getOutlinks(
+ "Test with http://www.nutch.org/index.html is it found? "
+ + "What about www.google.com at http://www.google.de "
+ + "A longer URL could be http://www.sybit.com/solutions/portals.html",
+ conf);
+
Assert.assertTrue("Url not found!", outlinks.length == 3);
- Assert.assertEquals("Wrong URL", "http://www.nutch.org/index.html", outlinks[0].getToUrl());
- Assert.assertEquals("Wrong URL", "http://www.google.de", outlinks[1].getToUrl());
- Assert.assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
+ Assert.assertEquals("Wrong URL", "http://www.nutch.org/index.html",
+ outlinks[0].getToUrl());
+ Assert.assertEquals("Wrong URL", "http://www.google.de",
+ outlinks[1].getToUrl());
+ Assert.assertEquals("Wrong URL",
+ "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
}
-
+
@Test
public void testGetOutlinksHttp2() {
- Outlink[] outlinks = OutlinkExtractor.getOutlinks(
- "Test with http://www.nutch.org/index.html is it found? " +
- "What about www.google.com at http://www.google.de " +
- "A longer URL could be http://www.sybit.com/solutions/portals.html", "http://www.sybit.de", conf);
-
+ Outlink[] outlinks = OutlinkExtractor
+ .getOutlinks(
+ "Test with http://www.nutch.org/index.html is it found? "
+ + "What about www.google.com at http://www.google.de "
+ + "A longer URL could be http://www.sybit.com/solutions/portals.html",
+ "http://www.sybit.de", conf);
+
Assert.assertTrue("Url not found!", outlinks.length == 3);
- Assert.assertEquals("Wrong URL", "http://www.nutch.org/index.html", outlinks[0].getToUrl());
- Assert.assertEquals("Wrong URL", "http://www.google.de", outlinks[1].getToUrl());
- Assert.assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
+ Assert.assertEquals("Wrong URL", "http://www.nutch.org/index.html",
+ outlinks[0].getToUrl());
+ Assert.assertEquals("Wrong URL", "http://www.google.de",
+ outlinks[1].getToUrl());
+ Assert.assertEquals("Wrong URL",
+ "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
}
-
+
@Test
public void testGetOutlinksFtp() {
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
- "Test with ftp://www.nutch.org is it found? " +
- "What about www.google.com at ftp://www.google.de", conf);
-
- Assert.assertTrue("Url not found!", outlinks.length >1);
- Assert.assertEquals("Wrong URL", "ftp://www.nutch.org", outlinks[0].getToUrl());
- Assert.assertEquals("Wrong URL", "ftp://www.google.de", outlinks[1].getToUrl());
+ "Test with ftp://www.nutch.org is it found? "
+ + "What about www.google.com at ftp://www.google.de", conf);
+
+ Assert.assertTrue("Url not found!", outlinks.length > 1);
+ Assert.assertEquals("Wrong URL", "ftp://www.nutch.org",
+ outlinks[0].getToUrl());
+ Assert.assertEquals("Wrong URL", "ftp://www.google.de",
+ outlinks[1].getToUrl());
}
}
Modified: nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java Thu Jan 29 05:38:59 2015
@@ -31,30 +31,27 @@ public class TestParseData {
String title = "The Foo Page";
- Outlink[] outlinks = new Outlink[] {
- new Outlink("http://foo.com/", "Foo"),
- new Outlink("http://bar.com/", "Bar")
- };
+ Outlink[] outlinks = new Outlink[] { new Outlink("http://foo.com/", "Foo"),
+ new Outlink("http://bar.com/", "Bar") };
Metadata metaData = new Metadata();
metaData.add("Language", "en/us");
metaData.add("Charset", "UTF-8");
- ParseData r = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
-
+ ParseData r = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks,
+ metaData);
+
WritableTestUtils.testWritable(r, null);
}
@Test
public void testMaxOutlinks() throws Exception {
Outlink[] outlinks = new Outlink[128];
- for (int i=0; i<outlinks.length; i++) {
+ for (int i = 0; i < outlinks.length; i++) {
outlinks[i] = new Outlink("http://outlink.com/" + i, "Outlink" + i);
}
ParseData original = new ParseData(ParseStatus.STATUS_SUCCESS,
- "Max Outlinks Title",
- outlinks,
- new Metadata());
+ "Max Outlinks Title", outlinks, new Metadata());
ParseData data = (ParseData) WritableTestUtils.writeRead(original, null);
Assert.assertEquals(outlinks.length, data.getOutlinks().length);
}
Modified: nutch/trunk/src/test/org/apache/nutch/parse/TestParseText.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/parse/TestParseText.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/parse/TestParseText.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/parse/TestParseText.java Thu Jan 29 05:38:59 2015
@@ -22,7 +22,7 @@ import org.junit.Test;
/** Unit tests for ParseText. */
-public class TestParseText {
+public class TestParseText {
@Test
public void testParseText() throws Exception {
Modified: nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java Thu Jan 29 05:38:59 2015
@@ -27,25 +27,25 @@ import org.junit.Test;
/**
* Unit test for new parse plugin selection.
- *
+ *
* @author Sebastien Le Callonnec
* @version 1.0
*/
public class TestParserFactory {
-
+
private Configuration conf;
private ParserFactory parserFactory;
/** Inits the Test Case with the test parse-plugin file */
@Before
public void setUp() throws Exception {
- conf = NutchConfiguration.create();
- conf.set("plugin.includes", ".*");
- conf.set("parse.plugin.file",
- "org/apache/nutch/parse/parse-plugin-test.xml");
- parserFactory = new ParserFactory(conf);
+ conf = NutchConfiguration.create();
+ conf.set("plugin.includes", ".*");
+ conf.set("parse.plugin.file",
+ "org/apache/nutch/parse/parse-plugin-test.xml");
+ parserFactory = new ParserFactory(conf);
}
-
+
/** Unit test for <code>getExtensions(String)</code> method. */
@Test
public void testGetExtensions() throws Exception {
@@ -56,48 +56,50 @@ public class TestParserFactory {
ext = parserFactory.getExtensions("foo/bar").get(0);
Assert.assertEquals("parse-tika", ext.getDescriptor().getPluginId());
}
-
+
/** Unit test to check <code>getParsers</code> method */
@Test
public void testGetParsers() throws Exception {
- Parser [] parsers = parserFactory.getParsers("text/html", "http://foo.com");
+ Parser[] parsers = parserFactory.getParsers("text/html", "http://foo.com");
Assert.assertNotNull(parsers);
Assert.assertEquals(1, parsers.length);
- Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser",
- parsers[0].getClass().getName());
+ Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser", parsers[0]
+ .getClass().getName());
parsers = parserFactory.getParsers("text/html; charset=ISO-8859-1",
- "http://foo.com");
+ "http://foo.com");
Assert.assertNotNull(parsers);
Assert.assertEquals(1, parsers.length);
- Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser",
- parsers[0].getClass().getName());
-
+ Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser", parsers[0]
+ .getClass().getName());
+
parsers = parserFactory.getParsers("application/x-javascript",
- "http://foo.com");
+ "http://foo.com");
Assert.assertNotNull(parsers);
Assert.assertEquals(1, parsers.length);
- Assert.assertEquals("org.apache.nutch.parse.js.JSParseFilter",
- parsers[0].getClass().getName());
-
+ Assert.assertEquals("org.apache.nutch.parse.js.JSParseFilter", parsers[0]
+ .getClass().getName());
+
parsers = parserFactory.getParsers("text/plain", "http://foo.com");
Assert.assertNotNull(parsers);
Assert.assertEquals(1, parsers.length);
- Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser",
- parsers[0].getClass().getName());
-
+ Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser", parsers[0]
+ .getClass().getName());
+
Parser parser1 = parserFactory.getParsers("text/plain", "http://foo.com")[0];
Parser parser2 = parserFactory.getParsers("*", "http://foo.com")[0];
-
- Assert.assertEquals("Different instances!", parser1.hashCode(), parser2.hashCode());
-
- //test and make sure that the rss parser is loaded even though its plugin.xml
- //doesn't claim to support text/rss, only application/rss+xml
- parsers = parserFactory.getParsers("text/rss","http://foo.com");
+
+ Assert.assertEquals("Different instances!", parser1.hashCode(),
+ parser2.hashCode());
+
+ // test and make sure that the rss parser is loaded even though its
+ // plugin.xml
+ // doesn't claim to support text/rss, only application/rss+xml
+ parsers = parserFactory.getParsers("text/rss", "http://foo.com");
Assert.assertNotNull(parsers);
- Assert.assertEquals(1,parsers.length);
- Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser",
- parsers[0].getClass().getName());
+ Assert.assertEquals(1, parsers.length);
+ Assert.assertEquals("org.apache.nutch.parse.tika.TikaParser", parsers[0]
+ .getClass().getName());
}
-
+
}
Modified: nutch/trunk/src/test/org/apache/nutch/plugin/HelloWorldExtension.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/plugin/HelloWorldExtension.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/plugin/HelloWorldExtension.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/plugin/HelloWorldExtension.java Thu Jan 29 05:38:59 2015
@@ -24,8 +24,11 @@ package org.apache.nutch.plugin;
*/
public class HelloWorldExtension implements ITestExtension {
- /* (non-Javadoc)
- * @see org.apache.nutch.plugin.ITestExtension#testGetExtension(java.lang.String)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.nutch.plugin.ITestExtension#testGetExtension(java.lang.String)
*/
public String testGetExtension(String hello) {
return hello + " World";
Modified: nutch/trunk/src/test/org/apache/nutch/plugin/ITestExtension.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/plugin/ITestExtension.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/plugin/ITestExtension.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/plugin/ITestExtension.java Thu Jan 29 05:38:59 2015
@@ -15,11 +15,12 @@
* limitations under the License.
*/
package org.apache.nutch.plugin;
+
/**
* A Simple Test Extension Interface.
*
* @author joa23
- *
+ *
*/
public interface ITestExtension {
public String testGetExtension(String hello);
Modified: nutch/trunk/src/test/org/apache/nutch/plugin/SimpleTestPlugin.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/plugin/SimpleTestPlugin.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/plugin/SimpleTestPlugin.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/plugin/SimpleTestPlugin.java Thu Jan 29 05:38:59 2015
@@ -28,8 +28,8 @@ import org.apache.hadoop.conf.Configurat
public class SimpleTestPlugin extends Plugin {
/**
- * @param pDescriptor
- * @param conf
+ * @param pDescriptor
+ * @param conf
*/
public SimpleTestPlugin(PluginDescriptor pDescriptor, Configuration conf) {
@@ -55,4 +55,3 @@ public class SimpleTestPlugin extends Pl
}
}
-
Modified: nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java Thu Jan 29 05:38:59 2015
@@ -42,15 +42,15 @@ public class TestPluginSystem {
private int fPluginCount;
private LinkedList<File> fFolders = new LinkedList<File>();
- private Configuration conf ;
+ private Configuration conf;
private PluginRepository repository;
@Before
public void setUp() throws Exception {
this.conf = NutchConfiguration.create();
conf.set("plugin.includes", ".*");
- // String string = this.conf.get("plugin.includes", "");
- // conf.set("plugin.includes", string + "|Dummy*");
+ // String string = this.conf.get("plugin.includes", "");
+ // conf.set("plugin.includes", string + "|Dummy*");
fPluginCount = 5;
createDummyPlugins(fPluginCount);
this.repository = PluginRepository.get(conf);
@@ -86,8 +86,7 @@ public class TestPluginSystem {
*/
@Test
public void testLoadPlugins() {
- PluginDescriptor[] descriptors = repository
- .getPluginDescriptors();
+ PluginDescriptor[] descriptors = repository.getPluginDescriptors();
int k = descriptors.length;
Assert.assertTrue(fPluginCount <= k);
for (int i = 0; i < descriptors.length; i++) {
@@ -123,11 +122,10 @@ public class TestPluginSystem {
@Test
public void testGetExtensionAndAttributes() {
String xpId = " sdsdsd";
- ExtensionPoint extensionPoint =repository
- .getExtensionPoint(xpId);
+ ExtensionPoint extensionPoint = repository.getExtensionPoint(xpId);
Assert.assertEquals(extensionPoint, null);
- Extension[] extension1 = repository
- .getExtensionPoint(getGetExtensionId()).getExtensions();
+ Extension[] extension1 = repository.getExtensionPoint(getGetExtensionId())
+ .getExtensions();
Assert.assertEquals(extension1.length, fPluginCount);
for (int i = 0; i < extension1.length; i++) {
Extension extension2 = extension1[i];
@@ -141,8 +139,8 @@ public class TestPluginSystem {
*/
@Test
public void testGetExtensionInstances() throws PluginRuntimeException {
- Extension[] extensions = repository
- .getExtensionPoint(getGetExtensionId()).getExtensions();
+ Extension[] extensions = repository.getExtensionPoint(getGetExtensionId())
+ .getExtensions();
Assert.assertEquals(extensions.length, fPluginCount);
for (int i = 0; i < extensions.length; i++) {
Extension extension = extensions[i];
@@ -161,8 +159,7 @@ public class TestPluginSystem {
*/
@Test
public void testGetClassLoader() {
- PluginDescriptor[] descriptors = repository
- .getPluginDescriptors();
+ PluginDescriptor[] descriptors = repository.getPluginDescriptors();
for (int i = 0; i < descriptors.length; i++) {
PluginDescriptor descriptor = descriptors[i];
Assert.assertNotNull(descriptor.getClassLoader());
@@ -174,8 +171,7 @@ public class TestPluginSystem {
*/
@Test
public void testGetResources() throws IOException {
- PluginDescriptor[] descriptors = repository
- .getPluginDescriptors();
+ PluginDescriptor[] descriptors = repository.getPluginDescriptors();
for (int i = 0; i < descriptors.length; i++) {
PluginDescriptor descriptor = descriptors[i];
if (!descriptor.getPluginId().startsWith("getPluginFolder()")) {
@@ -183,8 +179,7 @@ public class TestPluginSystem {
}
String value = descriptor.getResourceString("key", Locale.UK);
Assert.assertEquals("value", value);
- value = descriptor.getResourceString("key",
- Locale.TRADITIONAL_CHINESE);
+ value = descriptor.getResourceString("key", Locale.TRADITIONAL_CHINESE);
Assert.assertEquals("value", value);
}
@@ -199,7 +194,8 @@ public class TestPluginSystem {
Assert.fail("no plugin directory setuped..");
String name = strings[0];
- return new PluginManifestParser(conf, this.repository).getPluginFolder(name).toString();
+ return new PluginManifestParser(conf, this.repository)
+ .getPluginFolder(name).toString();
}
/**
@@ -213,8 +209,7 @@ public class TestPluginSystem {
File folder = new File(string);
folder.mkdir();
for (int i = 0; i < pCount; i++) {
- String pluginFolder = string + File.separator + "DummyPlugin"
- + i;
+ String pluginFolder = string + File.separator + "DummyPlugin" + i;
File file = new File(pluginFolder);
file.mkdir();
fFolders.add(file);
@@ -265,9 +260,8 @@ public class TestPluginSystem {
*/
private void createPluginManifest(int i, String pFolderPath)
throws IOException {
- FileWriter out = new FileWriter(pFolderPath + File.separator
- + "plugin.xml");
- String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ FileWriter out = new FileWriter(pFolderPath + File.separator + "plugin.xml");
+ String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<!--this is just a simple plugin for testing issues.-->"
+ "<plugin id=\"org.apache.nutch.plugin."
+ i
Modified: nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java Thu Jan 29 05:38:59 2015
@@ -58,52 +58,36 @@ public class TestContent {
Content c = null;
Metadata p = new Metadata();
- c = new Content("http://www.foo.com/",
- "http://www.foo.com/",
- "".getBytes("UTF8"),
- "text/html; charset=UTF-8", p, conf);
+ c = new Content("http://www.foo.com/", "http://www.foo.com/",
+ "".getBytes("UTF8"), "text/html; charset=UTF-8", p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/foo.html",
- "http://www.foo.com/",
- "".getBytes("UTF8"),
- "", p, conf);
+ c = new Content("http://www.foo.com/foo.html", "http://www.foo.com/",
+ "".getBytes("UTF8"), "", p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/foo.html",
- "http://www.foo.com/",
- "".getBytes("UTF8"),
- null, p, conf);
+ c = new Content("http://www.foo.com/foo.html", "http://www.foo.com/",
+ "".getBytes("UTF8"), null, p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/",
- "http://www.foo.com/",
- "<html></html>".getBytes("UTF8"),
- "", p, conf);
+ c = new Content("http://www.foo.com/", "http://www.foo.com/",
+ "<html></html>".getBytes("UTF8"), "", p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/foo.html",
- "http://www.foo.com/",
- "<html></html>".getBytes("UTF8"),
- "text/plain", p, conf);
+ c = new Content("http://www.foo.com/foo.html", "http://www.foo.com/",
+ "<html></html>".getBytes("UTF8"), "text/plain", p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/foo.png",
- "http://www.foo.com/",
- "<html></html>".getBytes("UTF8"),
- "text/plain", p, conf);
+ c = new Content("http://www.foo.com/foo.png", "http://www.foo.com/",
+ "<html></html>".getBytes("UTF8"), "text/plain", p, conf);
Assert.assertEquals("text/html", c.getContentType());
- c = new Content("http://www.foo.com/",
- "http://www.foo.com/",
- "".getBytes("UTF8"),
- "", p, conf);
+ c = new Content("http://www.foo.com/", "http://www.foo.com/",
+ "".getBytes("UTF8"), "", p, conf);
Assert.assertEquals(MimeTypes.OCTET_STREAM, c.getContentType());
- c = new Content("http://www.foo.com/",
- "http://www.foo.com/",
- "".getBytes("UTF8"),
- null, p, conf);
+ c = new Content("http://www.foo.com/", "http://www.foo.com/",
+ "".getBytes("UTF8"), null, p, conf);
Assert.assertNotNull(c.getContentType());
}
Modified: nutch/trunk/src/test/org/apache/nutch/protocol/TestProtocolFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/protocol/TestProtocolFactory.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/protocol/TestProtocolFactory.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/protocol/TestProtocolFactory.java Thu Jan 29 05:38:59 2015
@@ -27,58 +27,59 @@ public class TestProtocolFactory {
Configuration conf;
ProtocolFactory factory;
-
+
@Before
public void setUp() throws Exception {
conf = NutchConfiguration.create();
conf.set("plugin.includes", ".*");
conf.set("http.agent.name", "test-bot");
- factory=new ProtocolFactory(conf);
+ factory = new ProtocolFactory(conf);
}
@Test
- public void testGetProtocol(){
+ public void testGetProtocol() {
- //non existing protocol
+ // non existing protocol
try {
factory.getProtocol("xyzxyz://somehost");
Assert.fail("Must throw ProtocolNotFound");
} catch (ProtocolNotFound e) {
- //all is ok
- } catch (Exception ex){
+ // all is ok
+ } catch (Exception ex) {
Assert.fail("Must not throw any other exception");
}
-
- Protocol httpProtocol=null;
-
- //existing protocol
+
+ Protocol httpProtocol = null;
+
+ // existing protocol
try {
- httpProtocol=factory.getProtocol("http://somehost");
+ httpProtocol = factory.getProtocol("http://somehost");
Assert.assertNotNull(httpProtocol);
- } catch (Exception ex){
+ } catch (Exception ex) {
Assert.fail("Must not throw any other exception");
}
- //cache key
- Object protocol = ObjectCache.get(conf).getObject(Protocol.X_POINT_ID + "http");
+ // cache key
+ Object protocol = ObjectCache.get(conf).getObject(
+ Protocol.X_POINT_ID + "http");
Assert.assertNotNull(protocol);
Assert.assertEquals(httpProtocol, protocol);
-
- //test same object instance
+
+ // test same object instance
try {
- Assert.assertTrue(httpProtocol==factory.getProtocol("http://somehost"));
+ Assert.assertTrue(httpProtocol == factory.getProtocol("http://somehost"));
} catch (ProtocolNotFound e) {
Assert.fail("Must not throw any exception");
}
}
-
+
@Test
- public void testContains(){
+ public void testContains() {
Assert.assertTrue(factory.contains("http", "http"));
Assert.assertTrue(factory.contains("http", "http,ftp"));
Assert.assertTrue(factory.contains("http", " http , ftp"));
Assert.assertTrue(factory.contains("smb", "ftp,smb,http"));
Assert.assertFalse(factory.contains("smb", "smbb"));
}
-
+
}
Modified: nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java?rev=1655526&r1=1655525&r2=1655526&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java Thu Jan 29 05:38:59 2015
@@ -40,12 +40,13 @@ public class TestSegmentMerger {
Path seg2;
Path out;
int countSeg1, countSeg2;
-
+
@Before
public void setUp() throws Exception {
conf = NutchConfiguration.create();
fs = FileSystem.get(conf);
- testDir = new Path(conf.get("hadoop.tmp.dir"), "merge-" + System.currentTimeMillis());
+ testDir = new Path(conf.get("hadoop.tmp.dir"), "merge-"
+ + System.currentTimeMillis());
seg1 = new Path(testDir, "seg1");
seg2 = new Path(testDir, "seg2");
out = new Path(testDir, "out");
@@ -55,12 +56,13 @@ public class TestSegmentMerger {
DecimalFormat df = new DecimalFormat("0000000");
Text k = new Text();
Path ptPath = new Path(new Path(seg1, ParseText.DIR_NAME), "part-00000");
- MapFile.Writer w = new MapFile.Writer(conf, fs, ptPath.toString(), Text.class, ParseText.class);
+ MapFile.Writer w = new MapFile.Writer(conf, fs, ptPath.toString(),
+ Text.class, ParseText.class);
long curSize = 0;
countSeg1 = 0;
FileStatus fileStatus = fs.getFileStatus(ptPath);
long blkSize = fileStatus.getBlockSize();
-
+
while (curSize < blkSize * 2) {
k.set("seg1-" + df.format(countSeg1));
w.append(k, new ParseText("seg1 text " + countSeg1));
@@ -71,7 +73,8 @@ public class TestSegmentMerger {
System.err.println(" - done: " + countSeg1 + " records.");
System.err.println("Creating large segment 2...");
ptPath = new Path(new Path(seg2, ParseText.DIR_NAME), "part-00000");
- w = new MapFile.Writer(conf, fs, ptPath.toString(), Text.class, ParseText.class);
+ w = new MapFile.Writer(conf, fs, ptPath.toString(), Text.class,
+ ParseText.class);
curSize = 0;
countSeg2 = 0;
while (curSize < blkSize * 2) {
@@ -83,16 +86,16 @@ public class TestSegmentMerger {
w.close();
System.err.println(" - done: " + countSeg2 + " records.");
}
-
+
@After
public void tearDown() throws Exception {
fs.delete(testDir, true);
}
-
+
@Test
public void testLargeMerge() throws Exception {
SegmentMerger merger = new SegmentMerger(conf);
- merger.merge(out, new Path[]{seg1, seg2}, false, false, -1);
+ merger.merge(out, new Path[] { seg1, seg2 }, false, false, -1);
// verify output
FileStatus[] stats = fs.listStatus(out);
// there should be just one path
@@ -100,7 +103,8 @@ public class TestSegmentMerger {
Path outSeg = stats[0].getPath();
Text k = new Text();
ParseText v = new ParseText();
- MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, new Path(outSeg, ParseText.DIR_NAME), conf);
+ MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, new Path(
+ outSeg, ParseText.DIR_NAME), conf);
int cnt1 = 0, cnt2 = 0;
for (MapFile.Reader r : readers) {
while (r.next(k, v)) {