You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by le...@apache.org on 2015/01/09 07:34:37 UTC
svn commit: r1650447 [19/25] - in /nutch/branches/2.x: ./
src/java/org/apache/nutch/api/ src/java/org/apache/nutch/api/impl/
src/java/org/apache/nutch/api/impl/db/
src/java/org/apache/nutch/api/model/response/
src/java/org/apache/nutch/api/resources/ s...
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java Fri Jan 9 06:34:33 2015
@@ -46,377 +46,342 @@ import java.util.StringTokenizer;
*/
public class DOMContentUtilsTest {
- private static final String[] testPages = {
- // 0.
- new String(
- "<html><head><title> title </title><script> script </script>"
- + "</head><body> body <a href=\"http://www.nutch.org\">"
- + " anchor </a><!--comment-->" + "</body></html>"),
- // 1.
- new String(
- "<html><head><title> title </title><script> script </script>"
- + "</head><body> body <a href=\"/\">"
- + " home </a><!--comment-->"
- + "<style> style </style>"
- + " <a href=\"bot.html\">" + " bots </a>"
- + "</body></html>"),
- // 2.
- new String("<html><head><title> </title>" + "</head><body> "
- + "<a href=\"/\"> separate this "
- + "<a href=\"ok\"> from this" + "</a></a>"
- + "</body></html>"),
- // 3.
- // this one relies on certain neko fixup behavior, possibly
- // distributing the anchors into the LI's-but not the other
- // anchors (outside of them, instead)! So you get a tree that
- // looks like:
- // ... <li> <a href=/> home </a> </li>
- // <li> <a href=/> <a href="1"> 1 </a> </a> </li>
- // <li> <a href=/> <a href="1"> <a href="2"> 2 </a> </a> </a> </li>
- new String("<html><head><title> my title </title>"
- + "</head><body> body " + "<ul>"
- + "<li> <a href=\"/\"> home" + "<li> <a href=\"1\"> 1"
- + "<li> <a href=\"2\"> 2" + "</ul>" + "</body></html>"),
- // 4.
- // test frameset link extraction. The invalid frame in the middle
- // will be
- // fixed to a third standalone frame.
- new String("<html><head><title> my title </title>"
- + "</head><frameset rows=\"20,*\"> "
- + "<frame src=\"top.html\">" + "</frame>"
- + "<frameset cols=\"20,*\">" + "<frame src=\"left.html\">"
- + "</frame>" + "<frame src=\"invalid.html\"/>" + "</frame>"
- + "<frame src=\"right.html\">" + "</frame>" + "</frameset>"
- + "</frameset>" + "</body></html>"),
- // 5.
- // test <area> and <iframe> link extraction + url normalization
- new String(
- "<html><head><title> my title </title>"
- + "</head><body>"
- + "<img src=\"logo.gif\" usemap=\"#green\" border=\"0\">"
- + "<map name=\"green\">"
- + "<area shape=\"polygon\" coords=\"19,44,45,11,87\" href=\"../index.html\">"
- + "<area shape=\"rect\" coords=\"128,132,241,179\" href=\"#bottom\">"
- + "<area shape=\"circle\" coords=\"68,211,35\" href=\"../bot.html\">"
- + "</map>"
- + "<a name=\"bottom\"/><h1> the bottom </h1> "
- + "<iframe src=\"../docs/index.html\"/>"
- + "</body></html>"),
- // 6.
- // test whitespace processing for plain text extraction
- new String(
- "<html><head>\n <title> my\t\n title\r\n </title>\n"
- + " </head>\n"
- + " <body>\n"
- + " <h1> Whitespace\ttest </h1> \n"
- + "\t<a href=\"../index.html\">\n \twhitespace test\r\n\t</a> \t\n"
- + " <p> This is<span> a whitespace<span></span> test</span>. Newlines\n"
- + "should appear as space too.</p><p>Tabs\tare spaces too.\n</p>"
- + " This\t<b>is a</b> break -><br>and the line after<i> break</i>.<br>\n"
- + "<table>"
- + " <tr><td>one</td><td>two</td><td>three</td></tr>\n"
- + " <tr><td>space here </td><td> space there</td><td>no space</td></tr>"
- + "\t<tr><td>one\r\ntwo</td><td>two\tthree</td><td>three\r\tfour</td></tr>\n"
- + "</table>put some text here<Br>and there."
- + "<h2>End\tthis\rmadness\n!</h2>\r\n"
- + " . . . ."
- + "</body> </html>"),
- // 7.
- // test that <a rel=nofollow> links are not returned
- new String(
- "<html><head></head><body>"
- + "<a href=\"http://www.nutch.org\" rel=\"nofollow\"> ignore </a>"
- + "<a rel=\"nofollow\" href=\"http://www.nutch.org\"> ignore </a>"
- + "</body></html>"),
- // 8.
- // test that POST form actions are skipped
- new String(
- "<html><head></head><body>"
- + "<form method='POST' action='/search.jsp'><input type=text>"
- + "<input type=submit><p>test1</p></form>"
- + "<form method='GET' action='/dummy.jsp'><input type=text>"
- + "<input type=submit><p>test2</p></form></body></html>"),
- // 9.
- // test that all form actions are skipped
- new String(
- "<html><head></head><body>"
- + "<form method='POST' action='/search.jsp'><input type=text>"
- + "<input type=submit><p>test1</p></form>"
- + "<form method='GET' action='/dummy.jsp'><input type=text>"
- + "<input type=submit><p>test2</p></form></body></html>"),
- // 10.
- new String("<html><head><title> title </title>" + "</head><body>"
- + "<a href=\";x\">anchor1</a>"
- + "<a href=\"g;x\">anchor2</a>"
- + "<a href=\"g;x?y#s\">anchor3</a>" + "</body></html>"),
- // 11.
- new String("<html><head><title> title </title>" + "</head><body>"
- + "<a href=\"g\">anchor1</a>"
- + "<a href=\"g?y#s\">anchor2</a>"
- + "<a href=\"?y=1\">anchor3</a>"
- + "<a href=\"?y=1#s\">anchor4</a>"
- + "<a href=\"?y=1;somethingelse\">anchor5</a>"
- + "</body></html>"), };
-
- private static int SKIP = 9;
-
- private static String[] testBaseHrefs = { "http://www.nutch.org",
- "http://www.nutch.org/docs/foo.html", "http://www.nutch.org/docs/",
- "http://www.nutch.org/docs/", "http://www.nutch.org/frames/",
- "http://www.nutch.org/maps/", "http://www.nutch.org/whitespace/",
- "http://www.nutch.org//", "http://www.nutch.org/",
- "http://www.nutch.org/", "http://www.nutch.org/",
- "http://www.nutch.org/;something" };
-
- private static final DocumentFragment testDOMs[] = new DocumentFragment[testPages.length];
-
- private static URL[] testBaseHrefURLs = new URL[testPages.length];
-
- private static final String[] answerText = {
- "body anchor",
- "body home bots",
- "separate this from this",
- "body home 1 2",
- "",
- "the bottom",
- "Whitespace test whitespace test "
- + "This is a whitespace test . Newlines should appear as space too. "
- + "Tabs are spaces too. This is a break -> and the line after break . "
- + "one two three space here space there no space "
- + "one two two three three four put some text here and there. "
- + "End this madness ! . . . .", "ignore ignore",
- "test1 test2", "test1 test2", "anchor1 anchor2 anchor3",
- "anchor1 anchor2 anchor3 anchor4 anchor5" };
-
- private static final String[] answerTitle = { "title", "title", "",
- "my title", "my title", "my title", "my title", "", "", "",
- "title", "title" };
-
- // note: should be in page-order
- private static Outlink[][] answerOutlinks;
-
- private static Configuration conf;
- private static DOMContentUtils utils = null;
-
- public static final Logger Logger = LoggerFactory.getLogger(DOMContentUtilsTest.class);
-
- public DOMContentUtilsTest(String name) {
- }
-
- private static void setup() throws Exception {
- conf = NutchConfiguration.create();
- conf.setBoolean("parser.html.form.use_action", true);
- utils = new DOMContentUtils(conf);
- TikaParser tikaParser = new TikaParser();
- tikaParser.setConf(conf);
- Parser parser = tikaParser.getTikaConfig().getParser("text/html");
- for (int i = 0; i < testPages.length; i++) {
- Metadata tikamd = new Metadata();
-
- HTMLDocumentImpl doc = new HTMLDocumentImpl();
- doc.setErrorChecking(false);
- DocumentFragment root = doc.createDocumentFragment();
- DOMBuilder domhandler = new DOMBuilder(doc, root);
- ParseContext context = new ParseContext();
- // to add once available in Tika
- //context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
- try {
- parser.parse(new ByteArrayInputStream(testPages[i].getBytes()),
- domhandler, tikamd, context);
- testBaseHrefURLs[i] = new URL(testBaseHrefs[i]);
- } catch (Exception e) {
- e.printStackTrace();
- fail("caught exception: " + e);
- }
- testDOMs[i] = root;
- LSSerializerImpl lsi = new LSSerializerImpl();
- System.out.println("input " + i + ": '" + testPages[i] + "'");
- System.out.println("output " + i + ": '" + lsi.writeToString(root)
- + "'");
-
- }
- answerOutlinks = new Outlink[][] {
- // 0
- { new Outlink("http://www.nutch.org", "anchor"), },
- // 1
- {
- new Outlink("http://www.nutch.org/", "home"),
- new Outlink("http://www.nutch.org/docs/bot.html",
- "bots"), },
- // 2
- {
- new Outlink("http://www.nutch.org/", "separate this"),
- new Outlink("http://www.nutch.org/docs/ok", "from this"), },
-
- // 3
- { new Outlink("http://www.nutch.org/", "home"),
- new Outlink("http://www.nutch.org/docs/1", "1"),
- new Outlink("http://www.nutch.org/docs/2", "2"), },
- // 4
- {
- new Outlink("http://www.nutch.org/frames/top.html", ""),
- new Outlink("http://www.nutch.org/frames/left.html", ""),
- new Outlink("http://www.nutch.org/frames/invalid.html",""),
- new Outlink("http://www.nutch.org/frames/right.html",""),
- },
- // 5
- {
- new Outlink("http://www.nutch.org/maps/logo.gif", ""),
- new Outlink("http://www.nutch.org/index.html", ""),
- new Outlink("http://www.nutch.org/maps/#bottom", ""),
- new Outlink("http://www.nutch.org/bot.html", ""),
- new Outlink("http://www.nutch.org/docs/index.html", "")
- },
- // 6
- { new Outlink("http://www.nutch.org/index.html",
- "whitespace test"),
- },
- // 7
- {},
- // 8
- { new Outlink("http://www.nutch.org/dummy.jsp", "test2"), },
- // 9
- {},
- // 10
- {
- new Outlink("http://www.nutch.org/;x", "anchor1"),
- new Outlink("http://www.nutch.org/g;x", "anchor2"),
- new Outlink("http://www.nutch.org/g;x?y#s", "anchor3")
- },
- // 11
- {
- // this is tricky - see RFC3986 section 5.4.1 example 7
- new Outlink("http://www.nutch.org/g","anchor1"),
- new Outlink("http://www.nutch.org/g?y#s", "anchor2"),
- new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
- new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
- new Outlink("http://www.nutch.org/;something?y=1;somethingelse", "anchor5") }
- };
-
- }
-
- private static boolean equalsIgnoreWhitespace(String s1, String s2) {
- StringTokenizer st1 = new StringTokenizer(s1);
- StringTokenizer st2 = new StringTokenizer(s2);
-
- while (st1.hasMoreTokens()) {
- if (!st2.hasMoreTokens()) {
- Logger.info("st1+ '" + st1.nextToken() + "'");
- return false;
- }
- String st1Token = st1.nextToken();
- String st2Token = st2.nextToken();
- if (!st1Token.equals(st2Token)) {
- Logger.info("st1:'" + st1Token + "' != st2:'" + st2Token + "'");
- return false;
- }
- }
- if (st2.hasMoreTokens()) {
- System.err.println("st2+ '" + st2.nextToken() + "'");
- return false;
- }
- return true;
- }
-
- @Test
- public void testGetText() throws Exception {
- if (testDOMs[0] == null)
- setup();
- for (int i = 0; i < testPages.length; i++) {
- StringBuffer sb = new StringBuffer();
- utils.getText(sb, testDOMs[i]);
- String text = sb.toString();
- assertTrue(
- "example " + i + " : expecting text: " + answerText[i]
- + System.getProperty("line.separator")
- + System.getProperty("line.separator")
- + "got text: " + text,
- equalsIgnoreWhitespace(answerText[i], text));
- }
- }
-
- // won't work with Tika - the title is stored in the metadata but
- // not put in the XHTML representation
- @Test
- public void testGetTitle() throws Exception {
- if (testDOMs[0] == null)
- setup();
- for (int i = 0; i < testPages.length; i++) {
- StringBuffer sb = new StringBuffer();
- utils.getTitle(sb, testDOMs[i]);
- String title = sb.toString();
- assertTrue(
- "example " + i + " : expecting title: " + answerTitle[i]
- + System.getProperty("line.separator")
- + System.getProperty("line.separator")
- + "got title: " + title,
- equalsIgnoreWhitespace(answerTitle[i], title));
- }
- }
-
- @Test
- public void testGetOutlinks() throws Exception {
- if (testDOMs[0] == null)
- setup();
- for (int i = 0; i < testPages.length; i++) {
- ArrayList<Outlink> outlinks = new ArrayList<Outlink>();
- if (i == SKIP) {
- conf.setBoolean("parser.html.form.use_action", false);
- utils.setConf(conf);
- } else {
- conf.setBoolean("parser.html.form.use_action", true);
- utils.setConf(conf);
- }
- utils.getOutlinks(testBaseHrefURLs[i], outlinks, testDOMs[i]);
- Outlink[] outlinkArr = new Outlink[outlinks.size()];
- outlinkArr = outlinks.toArray(outlinkArr);
- compareOutlinks(i, answerOutlinks[i], outlinkArr);
- }
- }
-
- private static final void appendOutlinks(StringBuffer sb, Outlink[] o) {
- for (int i = 0; i < o.length; i++) {
- sb.append(o[i].toString());
- sb.append(System.getProperty("line.separator"));
- }
- }
-
- private static final String outlinksString(Outlink[] o) {
- StringBuffer sb = new StringBuffer();
- appendOutlinks(sb, o);
- return sb.toString();
- }
-
- private static final void compareOutlinks(int test, Outlink[] o1,
- Outlink[] o2) {
- if (o1.length != o2.length) {
- assertTrue(
- "test " + test
- + ", got wrong number of outlinks (expecting "
- + o1.length + ", got " + o2.length + ")"
- + System.getProperty("line.separator") + "answer: "
- + System.getProperty("line.separator")
- + outlinksString(o1)
- + System.getProperty("line.separator") + "got: "
- + System.getProperty("line.separator")
- + outlinksString(o2)
- + System.getProperty("line.separator"), false);
- }
-
- for (int i = 0; i < o1.length; i++) {
- if (!o1[i].equals(o2[i])) {
- assertTrue(
- "test " + test + ", got wrong outlinks at position "
- + i + System.getProperty("line.separator")
- + "answer: "
- + System.getProperty("line.separator")
- + o1[i].toString()
- + System.getProperty("line.separator")
- + "got: "
- + System.getProperty("line.separator")
- + o2[i].toString(), false);
-
- }
- }
- }
+ private static final String[] testPages = {
+ // 0.
+ new String("<html><head><title> title </title><script> script </script>"
+ + "</head><body> body <a href=\"http://www.nutch.org\">"
+ + " anchor </a><!--comment-->" + "</body></html>"),
+ // 1.
+ new String("<html><head><title> title </title><script> script </script>"
+ + "</head><body> body <a href=\"/\">" + " home </a><!--comment-->"
+ + "<style> style </style>" + " <a href=\"bot.html\">" + " bots </a>"
+ + "</body></html>"),
+ // 2.
+ new String("<html><head><title> </title>" + "</head><body> "
+ + "<a href=\"/\"> separate this " + "<a href=\"ok\"> from this"
+ + "</a></a>" + "</body></html>"),
+ // 3.
+ // this one relies on certain neko fixup behavior, possibly
+ // distributing the anchors into the LI's-but not the other
+ // anchors (outside of them, instead)! So you get a tree that
+ // looks like:
+ // ... <li> <a href=/> home </a> </li>
+ // <li> <a href=/> <a href="1"> 1 </a> </a> </li>
+ // <li> <a href=/> <a href="1"> <a href="2"> 2 </a> </a> </a> </li>
+ new String("<html><head><title> my title </title>"
+ + "</head><body> body " + "<ul>" + "<li> <a href=\"/\"> home"
+ + "<li> <a href=\"1\"> 1" + "<li> <a href=\"2\"> 2" + "</ul>"
+ + "</body></html>"),
+ // 4.
+ // test frameset link extraction. The invalid frame in the middle
+ // will be
+ // fixed to a third standalone frame.
+ new String("<html><head><title> my title </title>"
+ + "</head><frameset rows=\"20,*\"> " + "<frame src=\"top.html\">"
+ + "</frame>" + "<frameset cols=\"20,*\">"
+ + "<frame src=\"left.html\">" + "</frame>"
+ + "<frame src=\"invalid.html\"/>" + "</frame>"
+ + "<frame src=\"right.html\">" + "</frame>" + "</frameset>"
+ + "</frameset>" + "</body></html>"),
+ // 5.
+ // test <area> and <iframe> link extraction + url normalization
+ new String(
+ "<html><head><title> my title </title>"
+ + "</head><body>"
+ + "<img src=\"logo.gif\" usemap=\"#green\" border=\"0\">"
+ + "<map name=\"green\">"
+ + "<area shape=\"polygon\" coords=\"19,44,45,11,87\" href=\"../index.html\">"
+ + "<area shape=\"rect\" coords=\"128,132,241,179\" href=\"#bottom\">"
+ + "<area shape=\"circle\" coords=\"68,211,35\" href=\"../bot.html\">"
+ + "</map>" + "<a name=\"bottom\"/><h1> the bottom </h1> "
+ + "<iframe src=\"../docs/index.html\"/>" + "</body></html>"),
+ // 6.
+ // test whitespace processing for plain text extraction
+ new String(
+ "<html><head>\n <title> my\t\n title\r\n </title>\n"
+ + " </head>\n"
+ + " <body>\n"
+ + " <h1> Whitespace\ttest </h1> \n"
+ + "\t<a href=\"../index.html\">\n \twhitespace test\r\n\t</a> \t\n"
+ + " <p> This is<span> a whitespace<span></span> test</span>. Newlines\n"
+ + "should appear as space too.</p><p>Tabs\tare spaces too.\n</p>"
+ + " This\t<b>is a</b> break -><br>and the line after<i> break</i>.<br>\n"
+ + "<table>"
+ + " <tr><td>one</td><td>two</td><td>three</td></tr>\n"
+ + " <tr><td>space here </td><td> space there</td><td>no space</td></tr>"
+ + "\t<tr><td>one\r\ntwo</td><td>two\tthree</td><td>three\r\tfour</td></tr>\n"
+ + "</table>put some text here<Br>and there."
+ + "<h2>End\tthis\rmadness\n!</h2>\r\n"
+ + " . . . ." + "</body> </html>"),
+ // 7.
+ // test that <a rel=nofollow> links are not returned
+ new String("<html><head></head><body>"
+ + "<a href=\"http://www.nutch.org\" rel=\"nofollow\"> ignore </a>"
+ + "<a rel=\"nofollow\" href=\"http://www.nutch.org\"> ignore </a>"
+ + "</body></html>"),
+ // 8.
+ // test that POST form actions are skipped
+ new String("<html><head></head><body>"
+ + "<form method='POST' action='/search.jsp'><input type=text>"
+ + "<input type=submit><p>test1</p></form>"
+ + "<form method='GET' action='/dummy.jsp'><input type=text>"
+ + "<input type=submit><p>test2</p></form></body></html>"),
+ // 9.
+ // test that all form actions are skipped
+ new String("<html><head></head><body>"
+ + "<form method='POST' action='/search.jsp'><input type=text>"
+ + "<input type=submit><p>test1</p></form>"
+ + "<form method='GET' action='/dummy.jsp'><input type=text>"
+ + "<input type=submit><p>test2</p></form></body></html>"),
+ // 10.
+ new String("<html><head><title> title </title>" + "</head><body>"
+ + "<a href=\";x\">anchor1</a>" + "<a href=\"g;x\">anchor2</a>"
+ + "<a href=\"g;x?y#s\">anchor3</a>" + "</body></html>"),
+ // 11.
+ new String("<html><head><title> title </title>" + "</head><body>"
+ + "<a href=\"g\">anchor1</a>" + "<a href=\"g?y#s\">anchor2</a>"
+ + "<a href=\"?y=1\">anchor3</a>" + "<a href=\"?y=1#s\">anchor4</a>"
+ + "<a href=\"?y=1;somethingelse\">anchor5</a>" + "</body></html>"), };
+
+ private static int SKIP = 9;
+
+ private static String[] testBaseHrefs = { "http://www.nutch.org",
+ "http://www.nutch.org/docs/foo.html", "http://www.nutch.org/docs/",
+ "http://www.nutch.org/docs/", "http://www.nutch.org/frames/",
+ "http://www.nutch.org/maps/", "http://www.nutch.org/whitespace/",
+ "http://www.nutch.org//", "http://www.nutch.org/",
+ "http://www.nutch.org/", "http://www.nutch.org/",
+ "http://www.nutch.org/;something" };
+
+ private static final DocumentFragment testDOMs[] = new DocumentFragment[testPages.length];
+
+ private static URL[] testBaseHrefURLs = new URL[testPages.length];
+
+ private static final String[] answerText = {
+ "body anchor",
+ "body home bots",
+ "separate this from this",
+ "body home 1 2",
+ "",
+ "the bottom",
+ "Whitespace test whitespace test "
+ + "This is a whitespace test . Newlines should appear as space too. "
+ + "Tabs are spaces too. This is a break -> and the line after break . "
+ + "one two three space here space there no space "
+ + "one two two three three four put some text here and there. "
+ + "End this madness ! . . . .", "ignore ignore", "test1 test2",
+ "test1 test2", "anchor1 anchor2 anchor3",
+ "anchor1 anchor2 anchor3 anchor4 anchor5" };
+
+ private static final String[] answerTitle = { "title", "title", "",
+ "my title", "my title", "my title", "my title", "", "", "", "title",
+ "title" };
+
+ // note: should be in page-order
+ private static Outlink[][] answerOutlinks;
+
+ private static Configuration conf;
+ private static DOMContentUtils utils = null;
+
+ public static final Logger Logger = LoggerFactory
+ .getLogger(DOMContentUtilsTest.class);
+
+ public DOMContentUtilsTest(String name) {
+ }
+
+ private static void setup() throws Exception {
+ conf = NutchConfiguration.create();
+ conf.setBoolean("parser.html.form.use_action", true);
+ utils = new DOMContentUtils(conf);
+ TikaParser tikaParser = new TikaParser();
+ tikaParser.setConf(conf);
+ Parser parser = tikaParser.getTikaConfig().getParser("text/html");
+ for (int i = 0; i < testPages.length; i++) {
+ Metadata tikamd = new Metadata();
+
+ HTMLDocumentImpl doc = new HTMLDocumentImpl();
+ doc.setErrorChecking(false);
+ DocumentFragment root = doc.createDocumentFragment();
+ DOMBuilder domhandler = new DOMBuilder(doc, root);
+ ParseContext context = new ParseContext();
+ // to add once available in Tika
+ // context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
+ try {
+ parser.parse(new ByteArrayInputStream(testPages[i].getBytes()),
+ domhandler, tikamd, context);
+ testBaseHrefURLs[i] = new URL(testBaseHrefs[i]);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("caught exception: " + e);
+ }
+ testDOMs[i] = root;
+ LSSerializerImpl lsi = new LSSerializerImpl();
+ System.out.println("input " + i + ": '" + testPages[i] + "'");
+ System.out.println("output " + i + ": '" + lsi.writeToString(root) + "'");
+
+ }
+ answerOutlinks = new Outlink[][] {
+ // 0
+ { new Outlink("http://www.nutch.org", "anchor"), },
+ // 1
+ { new Outlink("http://www.nutch.org/", "home"),
+ new Outlink("http://www.nutch.org/docs/bot.html", "bots"), },
+ // 2
+ { new Outlink("http://www.nutch.org/", "separate this"),
+ new Outlink("http://www.nutch.org/docs/ok", "from this"), },
+
+ // 3
+ { new Outlink("http://www.nutch.org/", "home"),
+ new Outlink("http://www.nutch.org/docs/1", "1"),
+ new Outlink("http://www.nutch.org/docs/2", "2"), },
+ // 4
+ { new Outlink("http://www.nutch.org/frames/top.html", ""),
+ new Outlink("http://www.nutch.org/frames/left.html", ""),
+ new Outlink("http://www.nutch.org/frames/invalid.html", ""),
+ new Outlink("http://www.nutch.org/frames/right.html", ""), },
+ // 5
+ { new Outlink("http://www.nutch.org/maps/logo.gif", ""),
+ new Outlink("http://www.nutch.org/index.html", ""),
+ new Outlink("http://www.nutch.org/maps/#bottom", ""),
+ new Outlink("http://www.nutch.org/bot.html", ""),
+ new Outlink("http://www.nutch.org/docs/index.html", "") },
+ // 6
+ { new Outlink("http://www.nutch.org/index.html", "whitespace test"), },
+ // 7
+ {},
+ // 8
+ { new Outlink("http://www.nutch.org/dummy.jsp", "test2"), },
+ // 9
+ {},
+ // 10
+ { new Outlink("http://www.nutch.org/;x", "anchor1"),
+ new Outlink("http://www.nutch.org/g;x", "anchor2"),
+ new Outlink("http://www.nutch.org/g;x?y#s", "anchor3") },
+ // 11
+ {
+ // this is tricky - see RFC3986 section 5.4.1 example 7
+ new Outlink("http://www.nutch.org/g", "anchor1"),
+ new Outlink("http://www.nutch.org/g?y#s", "anchor2"),
+ new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
+ new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
+ new Outlink("http://www.nutch.org/;something?y=1;somethingelse",
+ "anchor5") } };
+
+ }
+
+ private static boolean equalsIgnoreWhitespace(String s1, String s2) {
+ StringTokenizer st1 = new StringTokenizer(s1);
+ StringTokenizer st2 = new StringTokenizer(s2);
+
+ while (st1.hasMoreTokens()) {
+ if (!st2.hasMoreTokens()) {
+ Logger.info("st1+ '" + st1.nextToken() + "'");
+ return false;
+ }
+ String st1Token = st1.nextToken();
+ String st2Token = st2.nextToken();
+ if (!st1Token.equals(st2Token)) {
+ Logger.info("st1:'" + st1Token + "' != st2:'" + st2Token + "'");
+ return false;
+ }
+ }
+ if (st2.hasMoreTokens()) {
+ System.err.println("st2+ '" + st2.nextToken() + "'");
+ return false;
+ }
+ return true;
+ }
+
+ @Test
+ public void testGetText() throws Exception {
+ if (testDOMs[0] == null)
+ setup();
+ for (int i = 0; i < testPages.length; i++) {
+ StringBuffer sb = new StringBuffer();
+ utils.getText(sb, testDOMs[i]);
+ String text = sb.toString();
+ assertTrue(
+ "example " + i + " : expecting text: " + answerText[i]
+ + System.getProperty("line.separator")
+ + System.getProperty("line.separator") + "got text: " + text,
+ equalsIgnoreWhitespace(answerText[i], text));
+ }
+ }
+
+ // won't work with Tika - the title is stored in the metadata but
+ // not put in the XHTML representation
+ @Test
+ public void testGetTitle() throws Exception {
+ if (testDOMs[0] == null)
+ setup();
+ for (int i = 0; i < testPages.length; i++) {
+ StringBuffer sb = new StringBuffer();
+ utils.getTitle(sb, testDOMs[i]);
+ String title = sb.toString();
+ assertTrue(
+ "example " + i + " : expecting title: " + answerTitle[i]
+ + System.getProperty("line.separator")
+ + System.getProperty("line.separator") + "got title: " + title,
+ equalsIgnoreWhitespace(answerTitle[i], title));
+ }
+ }
+
+ @Test
+ public void testGetOutlinks() throws Exception {
+ if (testDOMs[0] == null)
+ setup();
+ for (int i = 0; i < testPages.length; i++) {
+ ArrayList<Outlink> outlinks = new ArrayList<Outlink>();
+ if (i == SKIP) {
+ conf.setBoolean("parser.html.form.use_action", false);
+ utils.setConf(conf);
+ } else {
+ conf.setBoolean("parser.html.form.use_action", true);
+ utils.setConf(conf);
+ }
+ utils.getOutlinks(testBaseHrefURLs[i], outlinks, testDOMs[i]);
+ Outlink[] outlinkArr = new Outlink[outlinks.size()];
+ outlinkArr = outlinks.toArray(outlinkArr);
+ compareOutlinks(i, answerOutlinks[i], outlinkArr);
+ }
+ }
+
+ private static final void appendOutlinks(StringBuffer sb, Outlink[] o) {
+ for (int i = 0; i < o.length; i++) {
+ sb.append(o[i].toString());
+ sb.append(System.getProperty("line.separator"));
+ }
+ }
+
+ private static final String outlinksString(Outlink[] o) {
+ StringBuffer sb = new StringBuffer();
+ appendOutlinks(sb, o);
+ return sb.toString();
+ }
+
+ private static final void compareOutlinks(int test, Outlink[] o1, Outlink[] o2) {
+ if (o1.length != o2.length) {
+ assertTrue(
+ "test " + test + ", got wrong number of outlinks (expecting "
+ + o1.length + ", got " + o2.length + ")"
+ + System.getProperty("line.separator") + "answer: "
+ + System.getProperty("line.separator") + outlinksString(o1)
+ + System.getProperty("line.separator") + "got: "
+ + System.getProperty("line.separator") + outlinksString(o2)
+ + System.getProperty("line.separator"), false);
+ }
+
+ for (int i = 0; i < o1.length; i++) {
+ if (!o1[i].equals(o2[i])) {
+ assertTrue(
+ "test " + test + ", got wrong outlinks at position " + i
+ + System.getProperty("line.separator") + "answer: "
+ + System.getProperty("line.separator") + o1[i].toString()
+ + System.getProperty("line.separator") + "got: "
+ + System.getProperty("line.separator") + o2[i].toString(),
+ false);
+
+ }
+ }
+ }
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java Fri Jan 9 06:34:33 2015
@@ -39,9 +39,9 @@ public class TestImageMetadata {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
- private String sampleDir = System.getProperty("test.data",".");
+ private String sampleDir = System.getProperty("test.data", ".");
// Make sure sample files are copied to "test.data" as specified in
- private String[] sampleFiles = {"nutch_logo_tm.gif"};
+ private String[] sampleFiles = { "nutch_logo_tm.gif" };
@Test
public void testIt() throws ProtocolException, ParseException, IOException {
@@ -50,32 +50,32 @@ public class TestImageMetadata {
Parse parse;
Configuration conf = NutchConfiguration.create();
MimeUtil mimeutil = new MimeUtil(conf);
-
+
for (int i = 0; i < sampleFiles.length; i++) {
urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
-
+
File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
byte[] bytes = new byte[(int) file.length()];
DataInputStream in = new DataInputStream(new FileInputStream(file));
in.readFully(bytes);
in.close();
-
+
WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
String mtype = mimeutil.getMimeType(file);
page.setContentType(new Utf8(mtype));
-
+
parse = new ParseUtil(conf).parse(urlString, page);
-
- //assert width
+
+ // assert width
ByteBuffer bbufW = page.getMetadata().get(new Utf8("width"));
byte[] byteArrayW = new byte[bbufW.remaining()];
bbufW.get(byteArrayW);
String width = new String(byteArrayW);
assertEquals("121", width);
-
- //assert height
+
+ // assert height
ByteBuffer bbufH = page.getMetadata().get(new Utf8("height"));
byte[] byteArrayH = new byte[bbufH.remaining()];
bbufH.get(byteArrayH);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java Fri Jan 9 06:34:33 2015
@@ -44,63 +44,63 @@ import static org.junit.Assert.assertTru
*/
public class TestMSWordParser {
- private String fileSeparator = System.getProperty("file.separator");
- // This system property is defined in ./src/plugin/build-plugin.xml
- private String sampleDir = System.getProperty("test.data", ".");
- // Make sure sample files are copied to "test.data" as specified in
- // ./src/plugin/parse-msword/build.xml during plugin compilation.
- // Check ./src/plugin/parse-msword/sample/README.txt for what they are.
- private String[] sampleFiles = { "word97.doc" };
-
- private String expectedText = "This is a sample doc file prepared for nutch.";
-
- private Configuration conf;
-
- @Before
- public void setUp() {
- conf = NutchConfiguration.create();
- conf.set("file.content.limit", "-1");
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/parse-msword/build.xml during plugin compilation.
+ // Check ./src/plugin/parse-msword/sample/README.txt for what they are.
+ private String[] sampleFiles = { "word97.doc" };
+
+ private String expectedText = "This is a sample doc file prepared for nutch.";
+
+ private Configuration conf;
+
+ @Before
+ public void setUp() {
+ conf = NutchConfiguration.create();
+ conf.set("file.content.limit", "-1");
+ }
+
+ public String getTextContent(String fileName) throws ProtocolException,
+ ParseException, IOException {
+ String urlString = sampleDir + fileSeparator + fileName;
+
+ File file = new File(urlString);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+ Parse parse;
+ WebPage page = WebPage.newBuilder().build();
+ page.setBaseUrl(new Utf8("file:" + urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ // set the content type?
+ MimeUtil mimeutil = new MimeUtil(conf);
+ String mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype));
+
+ parse = new ParseUtil(conf).parse("file:" + urlString, page);
+ return parse.getText();
+ }
+
+ @Test
+ public void testIt() throws ProtocolException, ParseException, IOException {
+ for (int i = 0; i < sampleFiles.length; i++) {
+ String found = getTextContent(sampleFiles[i]);
+ assertTrue("text found : '" + found + "'", found.startsWith(expectedText));
}
+ }
- public String getTextContent(String fileName) throws ProtocolException,
- ParseException, IOException {
- String urlString = sampleDir + fileSeparator + fileName;
-
- File file = new File(urlString);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
- Parse parse;
- WebPage page = WebPage.newBuilder().build();
- page.setBaseUrl(new Utf8("file:"+urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- // set the content type?
- MimeUtil mimeutil = new MimeUtil(conf);
- String mtype = mimeutil.getMimeType(file);
- page.setContentType(new Utf8(mtype));
-
- parse = new ParseUtil(conf).parse("file:"+urlString, page);
- return parse.getText();
- }
-
- @Test
- public void testIt() throws ProtocolException, ParseException, IOException {
- for (int i = 0; i < sampleFiles.length; i++) {
- String found = getTextContent(sampleFiles[i]);
- assertTrue("text found : '" + found + "'", found
- .startsWith(expectedText));
- }
- }
-
- @Test
- public void testOpeningDocs() throws ProtocolException, ParseException, IOException {
- String[] filenames = new File(sampleDir).list();
- for (int i = 0; i < filenames.length; i++) {
- if (filenames[i].endsWith(".doc") == false)
- continue;
- assertTrue("cann't read content of " + filenames[i],
- getTextContent(filenames[i]).length() > 0);
- }
+ @Test
+ public void testOpeningDocs() throws ProtocolException, ParseException,
+ IOException {
+ String[] filenames = new File(sampleDir).list();
+ for (int i = 0; i < filenames.length; i++) {
+ if (filenames[i].endsWith(".doc") == false)
+ continue;
+ assertTrue("cann't read content of " + filenames[i],
+ getTextContent(filenames[i]).length() > 0);
}
+ }
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java Fri Jan 9 06:34:33 2015
@@ -57,11 +57,11 @@ public class TestOOParser {
Parse parse;
Configuration conf = NutchConfiguration.create();
MimeUtil mimeutil = new MimeUtil(conf);
-
+
try {
// read the test string
FileInputStream fis = new FileInputStream(sampleDir + fileSeparator
- + sampleText);
+ + sampleText);
StringBuffer sb = new StringBuffer();
int len = 0;
InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
@@ -83,7 +83,7 @@ public class TestOOParser {
urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
if (sampleFiles[i].startsWith("ootest") == false)
- continue;
+ continue;
File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
byte[] bytes = new byte[(int) file.length()];
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java Fri Jan 9 06:34:33 2015
@@ -43,43 +43,43 @@ import static org.junit.Assert.assertTru
*/
public class TestPdfParser {
- private String fileSeparator = System.getProperty("file.separator");
- // This system property is defined in ./src/plugin/build-plugin.xml
- private String sampleDir = System.getProperty("test.data", ".");
- // Make sure sample files are copied to "test.data" as specified in
- // ./src/plugin/parse-pdf/build.xml during plugin compilation.
- // Check ./src/plugin/parse-pdf/sample/README.txt for what they are.
- private String[] sampleFiles = { "pdftest.pdf", "encrypted.pdf" };
-
- private String expectedText = "A VERY SMALL PDF FILE";
-
- @Test
- public void testIt() throws ProtocolException, ParseException, IOException {
- String urlString;
- Parse parse;
- Configuration conf = NutchConfiguration.create();
- MimeUtil mimeutil = new MimeUtil(conf);
-
- for (int i = 0; i < sampleFiles.length; i++) {
- urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
-
- File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
-
- WebPage page = WebPage.newBuilder().build();
- page.setBaseUrl(new Utf8(urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- String mtype = mimeutil.getMimeType(file);
- page.setContentType(new Utf8(mtype));
-
- parse = new ParseUtil(conf).parse(urlString, page);
-
- int index = parse.getText().indexOf(expectedText);
- assertTrue(index > 0);
- }
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/parse-pdf/build.xml during plugin compilation.
+ // Check ./src/plugin/parse-pdf/sample/README.txt for what they are.
+ private String[] sampleFiles = { "pdftest.pdf", "encrypted.pdf" };
+
+ private String expectedText = "A VERY SMALL PDF FILE";
+
+ @Test
+ public void testIt() throws ProtocolException, ParseException, IOException {
+ String urlString;
+ Parse parse;
+ Configuration conf = NutchConfiguration.create();
+ MimeUtil mimeutil = new MimeUtil(conf);
+
+ for (int i = 0; i < sampleFiles.length; i++) {
+ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
+
+ File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+
+ WebPage page = WebPage.newBuilder().build();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ String mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype));
+
+ parse = new ParseUtil(conf).parse(urlString, page);
+
+ int index = parse.getText().indexOf(expectedText);
+ assertTrue(index > 0);
}
+ }
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java Fri Jan 9 06:34:33 2015
@@ -68,7 +68,7 @@ public class TestRSSParser {
* </ul>
*/
@Test
- public void testIt()throws ProtocolException, ParseException, IOException {
+ public void testIt() throws ProtocolException, ParseException, IOException {
String urlString;
Parse parse;
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java Fri Jan 9 06:34:33 2015
@@ -44,50 +44,49 @@ import static org.junit.Assert.assertEqu
*/
public class TestRTFParser {
- private String fileSeparator = System.getProperty("file.separator");
- // This system property is defined in ./src/plugin/build-plugin.xml
- private String sampleDir = System.getProperty("test.data", ".");
- // Make sure sample files are copied to "test.data" as specified in
- // ./src/plugin/parse-rtf/build.xml during plugin compilation.
- // Check ./src/plugin/parse-rtf/sample/README.txt for what they are.
- private String rtfFile = "test.rtf";
-
- @Test
- public void testIt() throws ProtocolException, ParseException, IOException {
-
- String urlString;
- Parse parse;
- Configuration conf = NutchConfiguration.create();
- MimeUtil mimeutil = new MimeUtil(conf);
-
- urlString = "file:" + sampleDir + fileSeparator + rtfFile;
-
- File file = new File(sampleDir + fileSeparator + rtfFile);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
-
- WebPage page = WebPage.newBuilder().build();
- page.setBaseUrl(new Utf8(urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- String mtype = mimeutil.getMimeType(file);
- page.setContentType(new Utf8(mtype));
-
- parse = new ParseUtil(conf).parse(urlString, page);
-
- String title = parse.getTitle();
- String text = parse.getText();
- assertEquals("test rft document", title);
- //assertEquals("The quick brown fox jumps over the lazy dog", text.trim());
-
-
- // HOW DO WE GET THE PARSE METADATA?
- // Metadata meta = parse();
-
- // METADATA extraction is not yet supported in Tika
- //
- // assertEquals("tests", meta.get(DublinCore.SUBJECT));
- }
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/parse-rtf/build.xml during plugin compilation.
+ // Check ./src/plugin/parse-rtf/sample/README.txt for what they are.
+ private String rtfFile = "test.rtf";
+
+ @Test
+ public void testIt() throws ProtocolException, ParseException, IOException {
+
+ String urlString;
+ Parse parse;
+ Configuration conf = NutchConfiguration.create();
+ MimeUtil mimeutil = new MimeUtil(conf);
+
+ urlString = "file:" + sampleDir + fileSeparator + rtfFile;
+
+ File file = new File(sampleDir + fileSeparator + rtfFile);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+
+ WebPage page = WebPage.newBuilder().build();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ String mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype));
+
+ parse = new ParseUtil(conf).parse(urlString, page);
+
+ String title = parse.getTitle();
+ String text = parse.getText();
+ assertEquals("test rft document", title);
+ // assertEquals("The quick brown fox jumps over the lazy dog", text.trim());
+
+ // HOW DO WE GET THE PARSE METADATA?
+ // Metadata meta = parse();
+
+ // METADATA extraction is not yet supported in Tika
+ //
+ // assertEquals("tests", meta.get(DublinCore.SUBJECT));
+ }
}
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java Fri Jan 9 06:34:33 2015
@@ -32,10 +32,11 @@ import java.util.Collection;
import java.util.HashSet;
/**
- * This class is a protocol plugin used for file: scheme.
- * It creates {@link FileResponse} object and gets the content of the url from it.
- * Configurable parameters are {@code file.content.limit} and {@code file.crawl.parent}
- * in nutch-default.xml defined under "file properties" section.
+ * This class is a protocol plugin used for file: scheme. It creates
+ * {@link FileResponse} object and gets the content of the url from it.
+ * Configurable parameters are {@code file.content.limit} and
+ * {@code file.crawl.parent} in nutch-default.xml defined under
+ * "file properties" section.
*/
public class File implements Protocol {
@@ -51,7 +52,7 @@ public class File implements Protocol {
static final int MAX_REDIRECTS = 5;
int maxContentLength;
-
+
boolean crawlParents;
/**
@@ -63,7 +64,8 @@ public class File implements Protocol {
private Configuration conf;
// constructor
- public File() { }
+ public File() {
+ }
/**
* Set the {@link Configuration} object
@@ -75,29 +77,32 @@ public class File implements Protocol {
this.symlinksAsRedirects = conf.getBoolean(
"file.crawl.redirect_noncanonical", true);
}
-
+
/**
* Get the {@link Configuration} object
*/
public Configuration getConf() {
return this.conf;
}
-
- /**
- * Set the point at which content is truncated.
+
+ /**
+ * Set the point at which content is truncated.
*/
public void setMaxContentLength(int maxContentLength) {
this.maxContentLength = maxContentLength;
}
-
- /**
- * Creates a {@link FileResponse} object corresponding to the url and
- * return a {@link ProtocolOutput} object as per the content received
+
+ /**
+ * Creates a {@link FileResponse} object corresponding to the url and return a
+ * {@link ProtocolOutput} object as per the content received
*
- * @param url Text containing the url
- * @param datum The CrawlDatum object corresponding to the url
+ * @param url
+ * Text containing the url
+ * @param datum
+ * The CrawlDatum object corresponding to the url
*
- * @return {@link ProtocolOutput} object for the content of the file indicated by url
+ * @return {@link ProtocolOutput} object for the content of the file indicated
+ * by url
*/
public ProtocolOutput getProtocolOutput(String url, WebPage page) {
String urlString = url.toString();
@@ -115,13 +120,16 @@ public class File implements Protocol {
return new ProtocolOutput(response.toContent()); // return it
} else if (code == 304) { // got not modified
- return new ProtocolOutput(response.toContent(), ProtocolStatusUtils.STATUS_NOTMODIFIED);
+ return new ProtocolOutput(response.toContent(),
+ ProtocolStatusUtils.STATUS_NOTMODIFIED);
} else if (code == 401) { // access denied / no read permissions
- return new ProtocolOutput(response.toContent(), ProtocolStatusUtils.makeStatus(ProtocolStatusUtils.ACCESS_DENIED));
+ return new ProtocolOutput(response.toContent(),
+ ProtocolStatusUtils.makeStatus(ProtocolStatusUtils.ACCESS_DENIED));
} else if (code == 404) { // no such file
- return new ProtocolOutput(response.toContent(), ProtocolStatusUtils.STATUS_NOTFOUND);
+ return new ProtocolOutput(response.toContent(),
+ ProtocolStatusUtils.STATUS_NOTFOUND);
} else if (code >= 300 && code < 400) { // handle redirect
u = new URL(response.getHeader("Location"));
@@ -156,8 +164,8 @@ public class File implements Protocol {
return FIELDS;
}
- /**
- * Quick way for running this class. Useful for debugging.
+ /**
+ * Quick way for running this class. Useful for debugging.
*/
public static void main(String[] args) throws Exception {
int maxContentLength = Integer.MIN_VALUE;
@@ -216,11 +224,11 @@ public class File implements Protocol {
file = null;
}
- /**
- * No robots parsing is done for file protocol.
- * So this returns a set of empty rules which will allow every url.
+ /**
+ * No robots parsing is done for file protocol. So this returns a set of empty
+ * rules which will allow every url.
*/
public BaseRobotRules getRobotRules(String url, WebPage page) {
return RobotRulesParser.EMPTY_RULES;
- }
+ }
}
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileError.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileError.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileError.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileError.java Fri Jan 9 06:34:33 2015
@@ -17,13 +17,16 @@
package org.apache.nutch.protocol.file;
-/** Thrown for File error codes.
+/**
+ * Thrown for File error codes.
*/
public class FileError extends FileException {
private int code;
-
- public int getCode(int code) { return code; }
+
+ public int getCode(int code) {
+ return code;
+ }
public FileError(int code) {
super("File Error: " + code);
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java Fri Jan 9 06:34:33 2015
@@ -30,31 +30,27 @@ import org.apache.nutch.protocol.Content
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
-
/************************************
- * FileResponse.java mimics file replies as http response.
- * It tries its best to follow http's way for headers, response codes
- * as well as exceptions.
- *
- * Comments:
- * (1) java.net.URL and java.net.URLConnection can handle file: scheme.
- * However they are not flexible enough, so not used in this implementation.
- *
- * (2) java.io.File is used for its abstractness across platforms.
- * Warning:
- * java.io.File API (1.4.2) does not elaborate on how special files,
- * such as /dev/* in unix and /proc/* on linux, are treated. Tests show
- * (a) java.io.File.isFile() return false for /dev/*
- * (b) java.io.File.isFile() return true for /proc/*
- * (c) java.io.File.length() return 0 for /proc/*
- * We are probably oaky for now. Could be buggy here.
- * How about special files on windows?
- *
- * (3) java.io.File API (1.4.2) does not seem to know unix hard link files.
- * They are just treated as individual files.
- *
+ * FileResponse.java mimics file replies as http response. It tries its best to
+ * follow http's way for headers, response codes as well as exceptions.
+ *
+ * Comments: (1) java.net.URL and java.net.URLConnection can handle file:
+ * scheme. However they are not flexible enough, so not used in this
+ * implementation.
+ *
+ * (2) java.io.File is used for its abstractness across platforms. Warning:
+ * java.io.File API (1.4.2) does not elaborate on how special files, such as
+ * /dev/* in unix and /proc/* on linux, are treated. Tests show (a)
+ * java.io.File.isFile() return false for /dev/* (b) java.io.File.isFile()
+ * return true for /proc/* (c) java.io.File.length() return 0 for /proc/* We are
+ * probably oaky for now. Could be buggy here. How about special files on
+ * windows?
+ *
+ * (3) java.io.File API (1.4.2) does not seem to know unix hard link files. They
+ * are just treated as individual files.
+ *
* (4) No funcy POSIX file attributes yet. May never need?
- *
+ *
* @author John Xing
***********************************/
public class FileResponse {
@@ -68,33 +64,36 @@ public class FileResponse {
private final File file;
private Configuration conf;
-
+
private MimeUtil MIME;
/** Returns the response code. */
- public int getCode() { return code; }
+ public int getCode() {
+ return code;
+ }
/** Returns the value of a named header. */
public String getHeader(String name) {
return headers.get(name);
}
- public byte[] getContent() { return content; }
+ public byte[] getContent() {
+ return content;
+ }
public Content toContent() {
return new Content(orig, base, (content != null ? content : EMPTY_CONTENT),
- getHeader(Response.CONTENT_TYPE),
- headers, this.conf);
+ getHeader(Response.CONTENT_TYPE), headers, this.conf);
}
-
+
public FileResponse(URL url, WebPage page, File file, Configuration conf)
- throws FileException, IOException {
+ throws FileException, IOException {
this.orig = url.toString();
this.base = url.toString();
this.file = file;
this.conf = conf;
-
+
MIME = new MimeUtil(conf);
if (!"file".equals(url.getProtocol()))
@@ -117,22 +116,22 @@ public class FileResponse {
path = java.net.URLDecoder.decode(path, "UTF-8");
} catch (UnsupportedEncodingException ex) {
}
-
+
try {
this.content = null;
// url.toURI() is only in j2se 1.5.0
- //java.io.File f = new java.io.File(url.toURI());
+ // java.io.File f = new java.io.File(url.toURI());
java.io.File f = new java.io.File(path);
if (!f.exists()) {
- this.code = 404; // http Not Found
+ this.code = 404; // http Not Found
return;
}
if (!f.canRead()) {
- this.code = 401; // http Unauthorized
+ this.code = 401; // http Unauthorized
return;
}
@@ -141,15 +140,17 @@ public class FileResponse {
// where case is insensitive
if (!f.equals(f.getCanonicalFile())) {
// set headers
- //hdrs.put("Location", f.getCanonicalFile().toURI());
- headers.set(Response.LOCATION, f.getCanonicalFile().toURI().toURL().toString());
+ // hdrs.put("Location", f.getCanonicalFile().toURI());
+ headers.set(Response.LOCATION, f.getCanonicalFile().toURI().toURL()
+ .toString());
- this.code = 300; // http redirect
+ this.code = 300; // http redirect
return;
}
if (f.lastModified() <= page.getModifiedTime()) {
this.code = 304;
- this.headers.set("Last-Modified", HttpDateFormat.toString(f.lastModified()));
+ this.headers.set("Last-Modified",
+ HttpDateFormat.toString(f.lastModified()));
return;
}
@@ -169,45 +170,46 @@ public class FileResponse {
}
// get file as http response
- private void getFileAsHttpResponse(java.io.File f)
- throws FileException, IOException {
+ private void getFileAsHttpResponse(java.io.File f) throws FileException,
+ IOException {
// ignore file of size larger than
// Integer.MAX_VALUE = 2^31-1 = 2147483647
long size = f.length();
if (size > Integer.MAX_VALUE) {
- throw new FileException("file is too large, size: "+size);
+ throw new FileException("file is too large, size: " + size);
// or we can do this?
- // this.code = 400; // http Bad request
+ // this.code = 400; // http Bad request
// return;
}
// capture content
int len = (int) size;
-
+
if (this.file.maxContentLength >= 0 && len > this.file.maxContentLength)
len = this.file.maxContentLength;
this.content = new byte[len];
java.io.InputStream is = new java.io.FileInputStream(f);
- int offset = 0; int n = 0;
+ int offset = 0;
+ int n = 0;
while (offset < len
- && (n = is.read(this.content, offset, len-offset)) >= 0) {
+ && (n = is.read(this.content, offset, len - offset)) >= 0) {
offset += n;
}
if (offset < len) { // keep whatever already have, but issue a warning
if (File.LOG.isWarnEnabled()) {
- File.LOG.warn("not enough bytes read from file: "+f.getPath());
+ File.LOG.warn("not enough bytes read from file: " + f.getPath());
}
}
- is.close();
+ is.close();
// set headers
headers.set(Response.CONTENT_LENGTH, new Long(size).toString());
- headers.set(Response.LAST_MODIFIED, HttpDateFormat.toString(f
- .lastModified()));
-
+ headers.set(Response.LAST_MODIFIED,
+ HttpDateFormat.toString(f.lastModified()));
+
String mimeType = MIME.getMimeType(f);
String mimeTypeString = mimeType != null ? mimeType.toString() : "";
headers.set(Response.CONTENT_TYPE, mimeTypeString);
@@ -217,33 +219,33 @@ public class FileResponse {
}
// get dir list as http response
- private void getDirAsHttpResponse(java.io.File f)
- throws IOException {
+ private void getDirAsHttpResponse(java.io.File f) throws IOException {
String path = f.toString();
if (this.file.crawlParents)
- this.content = list2html(f.listFiles(), path, "/".equals(path) ? false : true);
+ this.content = list2html(f.listFiles(), path, "/".equals(path) ? false
+ : true);
else
- this.content = list2html(f.listFiles(), path, false);
+ this.content = list2html(f.listFiles(), path, false);
// set headers
headers.set(Response.CONTENT_LENGTH,
- new Integer(this.content.length).toString());
+ new Integer(this.content.length).toString());
headers.set(Response.CONTENT_TYPE, "text/html");
headers.set(Response.LAST_MODIFIED,
- HttpDateFormat.toString(f.lastModified()));
+ HttpDateFormat.toString(f.lastModified()));
// response code
this.code = 200; // http OK
}
// generate html page from dir list
- private byte[] list2html(java.io.File[] list,
- String path, boolean includeDotDot) {
+ private byte[] list2html(java.io.File[] list, String path,
+ boolean includeDotDot) {
StringBuffer x = new StringBuffer("<html><head>");
- x.append("<title>Index of "+path+"</title></head>\n");
- x.append("<body><h1>Index of "+path+"</h1><pre>\n");
+ x.append("<title>Index of " + path + "</title></head>\n");
+ x.append("<body><h1>Index of " + path + "</h1><pre>\n");
if (includeDotDot) {
x.append("<a href='../'>../</a>\t-\t-\t-\n");
@@ -252,20 +254,20 @@ public class FileResponse {
// fix me: we might want to sort list here! but not now.
java.io.File f;
- for (int i=0; i<list.length; i++) {
+ for (int i = 0; i < list.length; i++) {
f = list[i];
String name = f.getName();
String time = HttpDateFormat.toString(f.lastModified());
if (f.isDirectory()) {
// java 1.4.2 api says dir itself and parent dir are not listed
// so the following is not needed.
- //if (name.equals(".") || name.equals(".."))
- // continue;
- x.append("<a href='"+name+"/"+"'>"+name+"/</a>\t");
- x.append(time+"\t-\n");
+ // if (name.equals(".") || name.equals(".."))
+ // continue;
+ x.append("<a href='" + name + "/" + "'>" + name + "/</a>\t");
+ x.append(time + "\t-\n");
} else if (f.isFile()) {
- x.append("<a href='"+name+ "'>"+name+"</a>\t");
- x.append(time+"\t"+f.length()+"\n");
+ x.append("<a href='" + name + "'>" + name + "</a>\t");
+ x.append(time + "\t" + f.length() + "\n");
} else {
// ignore any other
}
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java?rev=1650447&r1=1650446&r2=1650447&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java Fri Jan 9 06:34:33 2015
@@ -37,9 +37,10 @@ import static org.junit.Assert.*;
* @author mattmann
* @version $Revision$
*
- * <p>
- * Unit tests for the {@link File}Protocol.
- * </p>.
+ * <p>
+ * Unit tests for the {@link File}Protocol.
+ * </p>
+ * .
*/
public class TestProtocolFile {
@@ -47,12 +48,13 @@ public class TestProtocolFile {
private String sampleDir = System.getProperty("test.data", ".");
private static final String[] testTextFiles = new String[] {
- "testprotocolfile.txt", "testprotocolfile_(encoded).txt", "testprotocolfile_%28encoded%29.txt" };
+ "testprotocolfile.txt", "testprotocolfile_(encoded).txt",
+ "testprotocolfile_%28encoded%29.txt" };
private static final String expectedMimeType = "text/plain";
-
+
private Configuration conf;
-
+
@Before
public void setUp() {
conf = NutchConfiguration.create();
@@ -64,11 +66,11 @@ public class TestProtocolFile {
setContentType(testTextFile);
}
}
-
+
/**
- * Tests the setting of the <code>Response.CONTENT_TYPE</code> metadata
- * field.
- * @throws ProtocolNotFound
+ * Tests the setting of the <code>Response.CONTENT_TYPE</code> metadata field.
+ *
+ * @throws ProtocolNotFound
*
* @since NUTCH-384
*
@@ -78,19 +80,19 @@ public class TestProtocolFile {
assertNotNull(urlString);
WebPage datum = WebPage.newBuilder().build();
Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString);
- ProtocolOutput output = protocol.getProtocolOutput(urlString,datum);
+ ProtocolOutput output = protocol.getProtocolOutput(urlString, datum);
assertNotNull(output);
assertEquals("Status code: [" + output.getStatus().getCode()
+ "], not equal to: [" + ProtocolStatusCodes.SUCCESS + "]: args: ["
- + output.getStatus().getArgs() + "]", (Integer) ProtocolStatusCodes.SUCCESS, output
- .getStatus().getCode());
+ + output.getStatus().getArgs() + "]",
+ (Integer) ProtocolStatusCodes.SUCCESS, output.getStatus().getCode());
assertNotNull(output.getContent());
assertNotNull(output.getContent().getContentType());
assertEquals(expectedMimeType, output.getContent().getContentType());
assertNotNull(output.getContent().getMetadata());
- assertEquals(expectedMimeType, output.getContent().getMetadata().get(
- Response.CONTENT_TYPE));
+ assertEquals(expectedMimeType,
+ output.getContent().getMetadata().get(Response.CONTENT_TYPE));
}