You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2016/05/07 20:39:48 UTC
[1/6] nutch git commit: option to inlcude inlinks in commons crawl
dump
Repository: nutch
Updated Branches:
refs/heads/master 0e03daf11 -> 37458a93e
option to inlcude inlinks in commons crawl dump
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/6ff1ecbb
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/6ff1ecbb
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/6ff1ecbb
Branch: refs/heads/master
Commit: 6ff1ecbbb8527f760a81a316ffa6ae5b823f980a
Parents: 8572fd9
Author: Thamme Gowda <tg...@gmail.com>
Authored: Mon Apr 18 17:57:11 2016 -0700
Committer: Thamme Gowda <tg...@gmail.com>
Committed: Mon Apr 18 17:57:11 2016 -0700
----------------------------------------------------------------------
.../nutch/tools/AbstractCommonCrawlFormat.java | 18 ++++++++
.../nutch/tools/CommonCrawlDataDumper.java | 44 ++++++++++++++++----
.../apache/nutch/tools/CommonCrawlFormat.java | 15 +++++++
3 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/6ff1ecbb/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
index 3dc62c0..386ec4a 100644
--- a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
@@ -23,6 +23,7 @@ import java.net.InetAddress;
import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.text.ParseException;
+import java.util.List;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.util.URIUtil;
@@ -62,6 +63,8 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
protected String reverseKeyValue;
+ protected List<String> inLinks;
+
public AbstractCommonCrawlFormat(String url, Content content, Metadata metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException {
this.url = url;
this.content = content;
@@ -158,6 +161,13 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
// imported
writeKeyValue("imported", getImported());
+ if (getInLinks() != null){
+ startArray("inlinks", false, true);
+ for (String link : getInLinks()) {
+ writeArrayValue(link);
+ }
+ closeArray("inlinks", false, true);
+ }
closeObject(null);
return generateJson();
@@ -289,6 +299,14 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
return ifNullString(metadata.get("Content-Type"));
}
+ public List<String> getInLinks() {
+ return inLinks;
+ }
+
+ public void setInLinks(List<String> inLinks) {
+ this.inLinks = inLinks;
+ }
+
protected String getResponseDate() {
if (this.simpleDateFormat) {
String timestamp = null;
http://git-wip-us.apache.org/repos/asf/nutch/blob/6ff1ecbb/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index d00df0a..e26e088 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -58,13 +59,16 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.Inlink;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.nutch.crawl.LinkDbReader;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.segment.SegmentReader;
import org.apache.nutch.util.DumpFileUtil;
import org.apache.nutch.util.NutchConfiguration;
//Tika imports
@@ -222,13 +226,14 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
* @param outputDir the directory you wish to dump the raw content to. This
* directory will be created.
* @param segmentRootDir a directory containing one or more segments.
+ * @param linkdb Path to linkdb.
* @param gzip a boolean flag indicating whether the CBOR content should also
* be gzipped.
* @param epochFilename if {@code true}, output files will be names using the epoch time (in milliseconds).
* @param extension a file extension to use with output documents.
* @throws Exception if any exception occurs.
*/
- public void dump(File outputDir, File segmentRootDir, boolean gzip,
+ public void dump(File outputDir, File segmentRootDir, File linkdb, boolean gzip,
String[] mimeTypes, boolean epochFilename, String extension, boolean warc)
throws Exception {
if (gzip) {
@@ -257,6 +262,10 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
}
}
+ LinkDbReader linkDbReader = null;
+ if (linkdb != null) {
+ linkDbReader = new LinkDbReader(fs.getConf(), new Path(linkdb.toString()));
+ }
if (parts == null || parts.size() == 0) {
LOG.error( "No segment directories found in [ {}] ", segmentRootDir.getAbsolutePath());
System.exit(1);
@@ -346,11 +355,25 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
String mimeType = new Tika().detect(content.getContent());
// Maps file to JSON-based structure
- //TODO: Make this Jackson Format implementation reusable
- try (CommonCrawlFormat format = CommonCrawlFormatFactory
- .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
- jsonData = format.getJsonData(url, content, metadata);
+ List<String> inUrls = null;
+ if (linkDbReader != null) {
+ int max = 5000; //just in case there are too many urls!
+ Inlinks inlinks = linkDbReader.getInlinks((Text) key);
+ if (inlinks != null) {
+ Iterator<Inlink> iterator = inlinks.iterator();
+ inUrls = new ArrayList<>();
+ while (max >= 0 && iterator.hasNext()){
+ inUrls.add(iterator.next().getFromUrl());
+ max--;
+ }
}
+ }
+ //TODO: Make this Jackson Format implementation reusable
+ try (CommonCrawlFormat format = CommonCrawlFormatFactory
+ .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
+ format.setInLinks(inUrls);
+ jsonData = format.getJsonData(url, content, metadata);
+ }
collectStats(typeCounts, mimeType);
// collects statistics for the given mimetypes
@@ -587,6 +610,10 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
.withType(Number.class)
.withDescription("an optional file size in bytes for the WARC file(s)")
.create("warcSize");
+ Option linkDbOpt = OptionBuilder.withArgName("linkdb").hasArg(true)
+ .withDescription("an optional linkdb parameter to include inlinks in dump files")
+ .isRequired(false)
+ .create("linkdb");
// create the options
Options options = new Options();
@@ -606,6 +633,7 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
options.addOption(reverseKeyOpt);
options.addOption(extensionOpt);
options.addOption(sizeOpt);
+ options.addOption(linkDbOpt);
CommandLineParser parser = new GnuParser();
try {
@@ -635,6 +663,8 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
if (line.getParsedOptionValue("warcSize") != null) {
warcSize = (Long) line.getParsedOptionValue("warcSize");
}
+ String linkdbPath = line.getOptionValue("linkdb");
+ File linkdb = linkdbPath == null ? null : new File(linkdbPath);
CommonCrawlConfig config = new CommonCrawlConfig();
config.setKeyPrefix(keyPrefix);
@@ -655,7 +685,7 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
CommonCrawlDataDumper dumper = new CommonCrawlDataDumper(config);
- dumper.dump(outputDir, segmentRootDir, gzip, mimeTypes, epochFilename,
+ dumper.dump(outputDir, segmentRootDir, linkdb, gzip, mimeTypes, epochFilename,
extension, warc);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/nutch/blob/6ff1ecbb/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormat.java b/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
index 87baeb5..ea10e7d 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
@@ -23,6 +23,7 @@ import org.apache.nutch.protocol.Content;
import java.io.Closeable;
import java.io.IOException;
+import java.util.List;
/**
* Interface for all CommonCrawl formatter. It provides the signature for the
@@ -64,6 +65,20 @@ public interface CommonCrawlFormat extends Closeable {
public String getJsonData(String url, Content content, Metadata metadata,
ParseData parseData) throws IOException;
+
+ /**
+ * sets inlinks of this document
+ * @param inlinks list of inlinks
+ */
+ void setInLinks(List<String> inLinks);
+
+
+ /**
+ * gets set of inlinks
+ * @return gets inlinks of this document
+ */
+ List<String> getInLinks();
+
/**
* Optional method that could be implemented if the actual format needs some
* close procedure.
[3/6] nutch git commit: Fix code format issues
Posted by ma...@apache.org.
Fix code format issues
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/298cffc4
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/298cffc4
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/298cffc4
Branch: refs/heads/master
Commit: 298cffc4927eabf310ff65dd57847010ebb8f7f1
Parents: a088049
Author: Thamme Gowda <tg...@gmail.com>
Authored: Sat Apr 30 16:48:15 2016 -0700
Committer: Thamme Gowda <tg...@gmail.com>
Committed: Sat Apr 30 16:48:15 2016 -0700
----------------------------------------------------------------------
.../nutch/tools/AbstractCommonCrawlFormat.java | 4 +-
.../nutch/tools/CommonCrawlDataDumper.java | 43 ++++++++++----------
.../apache/nutch/tools/CommonCrawlFormat.java | 12 +++---
3 files changed, 30 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/298cffc4/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
index 386ec4a..d5a0154 100644
--- a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
@@ -79,7 +79,7 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
}
public String getJsonData(String url, Content content, Metadata metadata)
- throws IOException {
+ throws IOException {
this.url = url;
this.content = content;
this.metadata = metadata;
@@ -90,7 +90,7 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
public String getJsonData(String url, Content content, Metadata metadata,
ParseData parseData) throws IOException {
- // override of this is required in the actual formats
+ // override of this is required in the actual formats
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/nutch/blob/298cffc4/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index 83da679..5abd393 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -182,7 +182,8 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
private static final Logger LOG = LoggerFactory
.getLogger(CommonCrawlDataDumper.class.getName());
-
+ private static final int MAX_INLINKS = 5000;
+
private CommonCrawlConfig config = null;
// Gzip initialization
@@ -253,7 +254,8 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
//get all paths
List<Path> parts = new ArrayList<>();
RemoteIterator<LocatedFileStatus> files = fs.listFiles(segmentRootPath, true);
- String partPattern = ".*" + File.separator + Content.DIR_NAME + File.separator + "part-[0-9]{5}" + File.separator + "data";
+ String partPattern = ".*" + File.separator + Content.DIR_NAME
+ + File.separator + "part-[0-9]{5}" + File.separator + "data";
while (files.hasNext()) {
LocatedFileStatus next = files.next();
if (next.isFile()) {
@@ -269,7 +271,8 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
linkDbReader = new LinkDbReader(fs.getConf(), new Path(linkdb.toString()));
}
if (parts == null || parts.size() == 0) {
- LOG.error( "No segment directories found in [ {}] ", segmentRootDir.getAbsolutePath());
+ LOG.error( "No segment directories found in {} ",
+ segmentRootDir.getAbsolutePath());
System.exit(1);
}
LOG.info("Found {} segment parts", parts.size());
@@ -357,27 +360,25 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
String mimeType = new Tika().detect(content.getContent());
// Maps file to JSON-based structure
- Set<String> inUrls = null; ///may be there are duplicates, so using set
- if (linkDbReader != null) {
- int max = 5000; //just in case there are too many urls!
- Inlinks inlinks = linkDbReader.getInlinks((Text) key);
- if (inlinks != null) {
- Iterator<Inlink> iterator = inlinks.iterator();
- inUrls = new LinkedHashSet<>();
- while (max >= 0 && iterator.hasNext()){
- inUrls.add(iterator.next().getFromUrl());
- max--;
+ Set<String> inUrls = null; //there may be duplicates, so using set
+ if (linkDbReader != null) {
+ Inlinks inlinks = linkDbReader.getInlinks((Text) key);
+ if (inlinks != null) {
+ Iterator<Inlink> iterator = inlinks.iterator();
+ inUrls = new LinkedHashSet<>();
+ while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){
+ inUrls.add(iterator.next().getFromUrl());
+ }
}
}
- }
- //TODO: Make this Jackson Format implementation reusable
- try (CommonCrawlFormat format = CommonCrawlFormatFactory
- .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
- if (inUrls != null) {
- format.setInLinks(new ArrayList<>(inUrls));
+ //TODO: Make this Jackson Format implementation reusable
+ try (CommonCrawlFormat format = CommonCrawlFormatFactory
+ .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
+ if (inUrls != null) {
+ format.setInLinks(new ArrayList<>(inUrls));
+ }
+ jsonData = format.getJsonData(url, content, metadata);
}
- jsonData = format.getJsonData(url, content, metadata);
- }
collectStats(typeCounts, mimeType);
// collects statistics for the given mimetypes
http://git-wip-us.apache.org/repos/asf/nutch/blob/298cffc4/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlFormat.java b/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
index ea10e7d..0834d95 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlFormat.java
@@ -68,15 +68,15 @@ public interface CommonCrawlFormat extends Closeable {
/**
* sets inlinks of this document
- * @param inlinks list of inlinks
- */
+ * @param inLinks list of inlinks
+ */
void setInLinks(List<String> inLinks);
- /**
- * gets set of inlinks
- * @return gets inlinks of this document
- */
+ /**
+ * gets set of inlinks
+ * @return gets inlinks of this document
+ */
List<String> getInLinks();
/**
[6/6] nutch git commit: Record entry for GitHub-106: Option to
include inlinks in commonscrawl dump contributed by thammegowda.
Posted by ma...@apache.org.
Record entry for GitHub-106: Option to include inlinks in commonscrawl dump contributed by thammegowda.
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/37458a93
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/37458a93
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/37458a93
Branch: refs/heads/master
Commit: 37458a93e27a8c4562d34e83248549e064b07617
Parents: 04d203a
Author: Chris Mattmann <ma...@apache.org>
Authored: Sat May 7 10:39:40 2016 -1000
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sat May 7 10:39:40 2016 -1000
----------------------------------------------------------------------
CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/37458a93/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 436db07..1bb7377 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,8 @@ in the release announcement and keep it on top in this CHANGES.txt for the Nutch
Nutch Change Log
+* GitHub-106 Option to include inlinks in commonscrawl dump (thammegowda via mattmann)
+
* NUTCH-2256 Inconsistent log level (songwanging via snagel)
* NUTCH-2254 Indexer: character set issue with -addBinaryContent and -base64 (Federico Bonelli, snagel)
[2/6] nutch git commit: Handling duplicate inlinks
Posted by ma...@apache.org.
Handling duplicate inlinks
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/a0880491
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/a0880491
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/a0880491
Branch: refs/heads/master
Commit: a08804913c500b331c9296996ce1ba3d4929d3a8
Parents: 6ff1ecb
Author: Thamme Gowda <tg...@gmail.com>
Authored: Mon Apr 18 19:20:38 2016 -0700
Committer: Thamme Gowda <tg...@gmail.com>
Committed: Mon Apr 18 19:20:38 2016 -0700
----------------------------------------------------------------------
.../org/apache/nutch/tools/CommonCrawlDataDumper.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/a0880491/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index e26e088..83da679 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -33,8 +33,10 @@ import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -355,13 +357,13 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
String mimeType = new Tika().detect(content.getContent());
// Maps file to JSON-based structure
- List<String> inUrls = null;
+ Set<String> inUrls = null; ///may be there are duplicates, so using set
if (linkDbReader != null) {
int max = 5000; //just in case there are too many urls!
Inlinks inlinks = linkDbReader.getInlinks((Text) key);
if (inlinks != null) {
Iterator<Inlink> iterator = inlinks.iterator();
- inUrls = new ArrayList<>();
+ inUrls = new LinkedHashSet<>();
while (max >= 0 && iterator.hasNext()){
inUrls.add(iterator.next().getFromUrl());
max--;
@@ -371,7 +373,9 @@ public class CommonCrawlDataDumper extends Configured implements Tool {
//TODO: Make this Jackson Format implementation reusable
try (CommonCrawlFormat format = CommonCrawlFormatFactory
.getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
- format.setInLinks(inUrls);
+ if (inUrls != null) {
+ format.setInLinks(new ArrayList<>(inUrls));
+ }
jsonData = format.getJsonData(url, content, metadata);
}
[5/6] nutch git commit: Merge branch 'ccdump-inlinks' of
https://github.com/thammegowda/nutch into inlinks
Posted by ma...@apache.org.
Merge branch 'ccdump-inlinks' of https://github.com/thammegowda/nutch into inlinks
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/04d203ad
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/04d203ad
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/04d203ad
Branch: refs/heads/master
Commit: 04d203ad4d8fa66eab5f1bff53eedfe68b2d5310
Parents: 0e03daf f5adbcc
Author: Chris Mattmann <ma...@apache.org>
Authored: Sat May 7 10:38:50 2016 -1000
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sat May 7 10:38:50 2016 -1000
----------------------------------------------------------------------
.../nutch/tools/AbstractCommonCrawlFormat.java | 644 ++++++++++---------
.../nutch/tools/CommonCrawlDataDumper.java | 47 +-
.../apache/nutch/tools/CommonCrawlFormat.java | 15 +
3 files changed, 387 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
[4/6] nutch git commit: Code style : 2 spaces instead of tabs
Posted by ma...@apache.org.
Code style : 2 spaces instead of tabs
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/f5adbcc3
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/f5adbcc3
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/f5adbcc3
Branch: refs/heads/master
Commit: f5adbcc3c3bb447110b6733e0851b931e57465c3
Parents: 298cffc
Author: Thamme Gowda <tg...@gmail.com>
Authored: Sat Apr 30 17:15:50 2016 -0700
Committer: Thamme Gowda <tg...@gmail.com>
Committed: Sat Apr 30 17:15:50 2016 -0700
----------------------------------------------------------------------
.../nutch/tools/AbstractCommonCrawlFormat.java | 694 +++++++++----------
1 file changed, 347 insertions(+), 347 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/f5adbcc3/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
index d5a0154..1b425c4 100644
--- a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
@@ -43,351 +43,351 @@ import com.ibm.icu.text.SimpleDateFormat;
*
*/
public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
- protected static final Logger LOG = LoggerFactory.getLogger(AbstractCommonCrawlFormat.class.getName());
-
- protected String url;
-
- protected Content content;
-
- protected Metadata metadata;
-
- protected Configuration conf;
-
- protected String keyPrefix;
-
- protected boolean simpleDateFormat;
-
- protected boolean jsonArray;
-
- protected boolean reverseKey;
-
- protected String reverseKeyValue;
-
- protected List<String> inLinks;
-
- public AbstractCommonCrawlFormat(String url, Content content, Metadata metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException {
- this.url = url;
- this.content = content;
- this.metadata = metadata;
- this.conf = nutchConf;
-
- this.keyPrefix = config.getKeyPrefix();
- this.simpleDateFormat = config.getSimpleDateFormat();
- this.jsonArray = config.getJsonArray();
- this.reverseKey = config.getReverseKey();
- this.reverseKeyValue = config.getReverseKeyValue();
- }
-
- public String getJsonData(String url, Content content, Metadata metadata)
- throws IOException {
- this.url = url;
- this.content = content;
- this.metadata = metadata;
-
- return this.getJsonData();
- }
-
- public String getJsonData(String url, Content content, Metadata metadata,
- ParseData parseData) throws IOException {
-
- // override of this is required in the actual formats
- throw new NotImplementedException();
- }
-
- @Override
- public String getJsonData() throws IOException {
- try {
- startObject(null);
-
- // url
- writeKeyValue("url", getUrl());
-
- // timestamp
- writeKeyValue("timestamp", getTimestamp());
-
- // request
- startObject("request");
- writeKeyValue("method", getMethod());
- startObject("client");
- writeKeyValue("hostname", getRequestHostName());
- writeKeyValue("address", getRequestHostAddress());
- writeKeyValue("software", getRequestSoftware());
- writeKeyValue("robots", getRequestRobots());
- startObject("contact");
- writeKeyValue("name", getRequestContactName());
- writeKeyValue("email", getRequestContactEmail());
- closeObject("contact");
- closeObject("client");
- // start request headers
- startHeaders("headers", false, true);
- writeKeyValueWrapper("Accept", getRequestAccept());
- writeKeyValueWrapper("Accept-Encoding", getRequestAcceptEncoding());
- writeKeyValueWrapper("Accept-Language", getRequestAcceptLanguage());
- writeKeyValueWrapper("User-Agent", getRequestUserAgent());
- //closeObject("headers");
- closeHeaders("headers", false, true);
- writeKeyNull("body");
- closeObject("request");
-
- // response
- startObject("response");
- writeKeyValue("status", getResponseStatus());
- startObject("server");
- writeKeyValue("hostname", getResponseHostName());
- writeKeyValue("address", getResponseAddress());
- closeObject("server");
- // start response headers
- startHeaders("headers", false, true);
- writeKeyValueWrapper("Content-Encoding", getResponseContentEncoding());
- writeKeyValueWrapper("Content-Type", getResponseContentType());
- writeKeyValueWrapper("Date", getResponseDate());
- writeKeyValueWrapper("Server", getResponseServer());
- for (String name : metadata.names()) {
- if (name.equalsIgnoreCase("Content-Encoding") || name.equalsIgnoreCase("Content-Type") || name.equalsIgnoreCase("Date") || name.equalsIgnoreCase("Server")) {
- continue;
- }
- writeKeyValueWrapper(name, metadata.get(name));
- }
- closeHeaders("headers", false, true);
- writeKeyValue("body", getResponseContent());
- closeObject("response");
-
- // key
- if (!this.keyPrefix.isEmpty()) {
- this.keyPrefix += "-";
- }
- writeKeyValue("key", this.keyPrefix + getKey());
-
- // imported
- writeKeyValue("imported", getImported());
-
- if (getInLinks() != null){
- startArray("inlinks", false, true);
- for (String link : getInLinks()) {
- writeArrayValue(link);
- }
- closeArray("inlinks", false, true);
- }
- closeObject(null);
-
- return generateJson();
-
- } catch (IOException ioe) {
- LOG.warn("Error in processing file " + url + ": " + ioe.getMessage());
- throw new IOException("Error in generating JSON:" + ioe.getMessage());
- }
- }
-
- // abstract methods
-
- protected abstract void writeKeyValue(String key, String value) throws IOException;
-
- protected abstract void writeKeyNull(String key) throws IOException;
-
- protected abstract void startArray(String key, boolean nested, boolean newline) throws IOException;
-
- protected abstract void closeArray(String key, boolean nested, boolean newline) throws IOException;
-
- protected abstract void writeArrayValue(String value) throws IOException;
-
- protected abstract void startObject(String key) throws IOException;
-
- protected abstract void closeObject(String key) throws IOException;
-
- protected abstract String generateJson() throws IOException;
-
- // getters
-
- protected String getUrl() {
- try {
- return URIUtil.encodePath(url);
- } catch (URIException e) {
- LOG.error("Can't encode URL " + url);
- }
-
- return url;
- }
-
- protected String getTimestamp() {
- if (this.simpleDateFormat) {
- String timestamp = null;
- try {
- long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get(Metadata.LAST_MODIFIED))).getTime();
- timestamp = String.valueOf(epoch);
- } catch (ParseException pe) {
- LOG.warn(pe.getMessage());
- }
- return timestamp;
- } else {
- return ifNullString(metadata.get(Metadata.LAST_MODIFIED));
- }
- }
-
- protected String getMethod() {
- return new String("GET");
- }
-
- protected String getRequestHostName() {
- String hostName = "";
- try {
- hostName = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException uhe) {
-
- }
- return hostName;
- }
-
- protected String getRequestHostAddress() {
- String hostAddress = "";
- try {
- hostAddress = InetAddress.getLocalHost().getHostAddress();
- } catch (UnknownHostException uhe) {
-
- }
- return hostAddress;
- }
-
- protected String getRequestSoftware() {
- return conf.get("http.agent.version", "");
- }
-
- protected String getRequestRobots() {
- return new String("CLASSIC");
- }
-
- protected String getRequestContactName() {
- return conf.get("http.agent.name", "");
- }
-
- protected String getRequestContactEmail() {
- return conf.get("http.agent.email", "");
- }
-
- protected String getRequestAccept() {
- return conf.get("http.accept", "");
- }
-
- protected String getRequestAcceptEncoding() {
- return new String(""); // TODO
- }
-
- protected String getRequestAcceptLanguage() {
- return conf.get("http.accept.language", "");
- }
-
- protected String getRequestUserAgent() {
- return conf.get("http.robots.agents", "");
- }
-
- protected String getResponseStatus() {
- return ifNullString(metadata.get("status"));
- }
-
- protected String getResponseHostName() {
- return URLUtil.getHost(url);
- }
-
- protected String getResponseAddress() {
- return ifNullString(metadata.get("_ip_"));
- }
-
- protected String getResponseContentEncoding() {
- return ifNullString(metadata.get("Content-Encoding"));
- }
-
- protected String getResponseContentType() {
- return ifNullString(metadata.get("Content-Type"));
- }
-
- public List<String> getInLinks() {
- return inLinks;
- }
-
- public void setInLinks(List<String> inLinks) {
- this.inLinks = inLinks;
- }
-
- protected String getResponseDate() {
- if (this.simpleDateFormat) {
- String timestamp = null;
- try {
- long epoch = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get("Date"))).getTime();
- timestamp = String.valueOf(epoch);
- } catch (ParseException pe) {
- LOG.warn(pe.getMessage());
- }
- return timestamp;
- } else {
- return ifNullString(metadata.get("Date"));
- }
- }
-
- protected String getResponseServer() {
- return ifNullString(metadata.get("Server"));
- }
-
- protected String getResponseContent() {
- return new String(content.getContent());
- }
-
- protected String getKey() {
- if (this.reverseKey) {
- return this.reverseKeyValue;
- }
- else {
- return url;
- }
- }
-
- protected String getImported() {
- if (this.simpleDateFormat) {
- String timestamp = null;
- try {
- long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get("Date"))).getTime();
- timestamp = String.valueOf(epoch);
- } catch (ParseException pe) {
- LOG.warn(pe.getMessage());
- }
- return timestamp;
- } else {
- return ifNullString(metadata.get("Date"));
- }
- }
-
- private static String ifNullString(String value) {
- return (value != null) ? value : "";
- }
-
- private void startHeaders(String key, boolean nested, boolean newline) throws IOException {
- if (this.jsonArray) {
- startArray(key, nested, newline);
- }
- else {
- startObject(key);
- }
- }
-
- private void closeHeaders(String key, boolean nested, boolean newline) throws IOException {
- if (this.jsonArray) {
- closeArray(key, nested, newline);
- }
- else {
- closeObject(key);
- }
- }
-
- private void writeKeyValueWrapper(String key, String value) throws IOException {
- if (this.jsonArray) {
- startArray(null, true, false);
- writeArrayValue(key);
- writeArrayValue(value);
- closeArray(null, true, false);
- }
- else {
- writeKeyValue(key, value);
- }
- }
-
- @Override
- public void close() {}
+ protected static final Logger LOG = LoggerFactory.getLogger(AbstractCommonCrawlFormat.class.getName());
+
+ protected String url;
+
+ protected Content content;
+
+ protected Metadata metadata;
+
+ protected Configuration conf;
+
+ protected String keyPrefix;
+
+ protected boolean simpleDateFormat;
+
+ protected boolean jsonArray;
+
+ protected boolean reverseKey;
+
+ protected String reverseKeyValue;
+
+ protected List<String> inLinks;
+
+ public AbstractCommonCrawlFormat(String url, Content content, Metadata metadata, Configuration nutchConf, CommonCrawlConfig config) throws IOException {
+ this.url = url;
+ this.content = content;
+ this.metadata = metadata;
+ this.conf = nutchConf;
+
+ this.keyPrefix = config.getKeyPrefix();
+ this.simpleDateFormat = config.getSimpleDateFormat();
+ this.jsonArray = config.getJsonArray();
+ this.reverseKey = config.getReverseKey();
+ this.reverseKeyValue = config.getReverseKeyValue();
+ }
+
+ public String getJsonData(String url, Content content, Metadata metadata)
+ throws IOException {
+ this.url = url;
+ this.content = content;
+ this.metadata = metadata;
+
+ return this.getJsonData();
+ }
+
+ public String getJsonData(String url, Content content, Metadata metadata,
+ ParseData parseData) throws IOException {
+
+ // override of this is required in the actual formats
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJsonData() throws IOException {
+ try {
+ startObject(null);
+
+ // url
+ writeKeyValue("url", getUrl());
+
+ // timestamp
+ writeKeyValue("timestamp", getTimestamp());
+
+ // request
+ startObject("request");
+ writeKeyValue("method", getMethod());
+ startObject("client");
+ writeKeyValue("hostname", getRequestHostName());
+ writeKeyValue("address", getRequestHostAddress());
+ writeKeyValue("software", getRequestSoftware());
+ writeKeyValue("robots", getRequestRobots());
+ startObject("contact");
+ writeKeyValue("name", getRequestContactName());
+ writeKeyValue("email", getRequestContactEmail());
+ closeObject("contact");
+ closeObject("client");
+ // start request headers
+ startHeaders("headers", false, true);
+ writeKeyValueWrapper("Accept", getRequestAccept());
+ writeKeyValueWrapper("Accept-Encoding", getRequestAcceptEncoding());
+ writeKeyValueWrapper("Accept-Language", getRequestAcceptLanguage());
+ writeKeyValueWrapper("User-Agent", getRequestUserAgent());
+ //closeObject("headers");
+ closeHeaders("headers", false, true);
+ writeKeyNull("body");
+ closeObject("request");
+
+ // response
+ startObject("response");
+ writeKeyValue("status", getResponseStatus());
+ startObject("server");
+ writeKeyValue("hostname", getResponseHostName());
+ writeKeyValue("address", getResponseAddress());
+ closeObject("server");
+ // start response headers
+ startHeaders("headers", false, true);
+ writeKeyValueWrapper("Content-Encoding", getResponseContentEncoding());
+ writeKeyValueWrapper("Content-Type", getResponseContentType());
+ writeKeyValueWrapper("Date", getResponseDate());
+ writeKeyValueWrapper("Server", getResponseServer());
+ for (String name : metadata.names()) {
+ if (name.equalsIgnoreCase("Content-Encoding") || name.equalsIgnoreCase("Content-Type") || name.equalsIgnoreCase("Date") || name.equalsIgnoreCase("Server")) {
+ continue;
+ }
+ writeKeyValueWrapper(name, metadata.get(name));
+ }
+ closeHeaders("headers", false, true);
+ writeKeyValue("body", getResponseContent());
+ closeObject("response");
+
+ // key
+ if (!this.keyPrefix.isEmpty()) {
+ this.keyPrefix += "-";
+ }
+ writeKeyValue("key", this.keyPrefix + getKey());
+
+ // imported
+ writeKeyValue("imported", getImported());
+
+ if (getInLinks() != null){
+ startArray("inlinks", false, true);
+ for (String link : getInLinks()) {
+ writeArrayValue(link);
+ }
+ closeArray("inlinks", false, true);
+ }
+ closeObject(null);
+
+ return generateJson();
+
+ } catch (IOException ioe) {
+ LOG.warn("Error in processing file " + url + ": " + ioe.getMessage());
+ throw new IOException("Error in generating JSON:" + ioe.getMessage());
+ }
+ }
+
+ // abstract methods
+
+ protected abstract void writeKeyValue(String key, String value) throws IOException;
+
+ protected abstract void writeKeyNull(String key) throws IOException;
+
+ protected abstract void startArray(String key, boolean nested, boolean newline) throws IOException;
+
+ protected abstract void closeArray(String key, boolean nested, boolean newline) throws IOException;
+
+ protected abstract void writeArrayValue(String value) throws IOException;
+
+ protected abstract void startObject(String key) throws IOException;
+
+ protected abstract void closeObject(String key) throws IOException;
+
+ protected abstract String generateJson() throws IOException;
+
+ // getters
+
+ protected String getUrl() {
+ try {
+ return URIUtil.encodePath(url);
+ } catch (URIException e) {
+ LOG.error("Can't encode URL " + url);
+ }
+
+ return url;
+ }
+
+ protected String getTimestamp() {
+ if (this.simpleDateFormat) {
+ String timestamp = null;
+ try {
+ long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get(Metadata.LAST_MODIFIED))).getTime();
+ timestamp = String.valueOf(epoch);
+ } catch (ParseException pe) {
+ LOG.warn(pe.getMessage());
+ }
+ return timestamp;
+ } else {
+ return ifNullString(metadata.get(Metadata.LAST_MODIFIED));
+ }
+ }
+
+ protected String getMethod() {
+ return new String("GET");
+ }
+
+ protected String getRequestHostName() {
+ String hostName = "";
+ try {
+ hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException uhe) {
+
+ }
+ return hostName;
+ }
+
+ protected String getRequestHostAddress() {
+ String hostAddress = "";
+ try {
+ hostAddress = InetAddress.getLocalHost().getHostAddress();
+ } catch (UnknownHostException uhe) {
+
+ }
+ return hostAddress;
+ }
+
+ protected String getRequestSoftware() {
+ return conf.get("http.agent.version", "");
+ }
+
+ protected String getRequestRobots() {
+ return new String("CLASSIC");
+ }
+
+ protected String getRequestContactName() {
+ return conf.get("http.agent.name", "");
+ }
+
+ protected String getRequestContactEmail() {
+ return conf.get("http.agent.email", "");
+ }
+
+ protected String getRequestAccept() {
+ return conf.get("http.accept", "");
+ }
+
+ protected String getRequestAcceptEncoding() {
+ return new String(""); // TODO
+ }
+
+ protected String getRequestAcceptLanguage() {
+ return conf.get("http.accept.language", "");
+ }
+
+ protected String getRequestUserAgent() {
+ return conf.get("http.robots.agents", "");
+ }
+
+ protected String getResponseStatus() {
+ return ifNullString(metadata.get("status"));
+ }
+
+ protected String getResponseHostName() {
+ return URLUtil.getHost(url);
+ }
+
+ protected String getResponseAddress() {
+ return ifNullString(metadata.get("_ip_"));
+ }
+
+ protected String getResponseContentEncoding() {
+ return ifNullString(metadata.get("Content-Encoding"));
+ }
+
+ protected String getResponseContentType() {
+ return ifNullString(metadata.get("Content-Type"));
+ }
+
+ public List<String> getInLinks() {
+ return inLinks;
+ }
+
+ public void setInLinks(List<String> inLinks) {
+ this.inLinks = inLinks;
+ }
+
+ protected String getResponseDate() {
+ if (this.simpleDateFormat) {
+ String timestamp = null;
+ try {
+ long epoch = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get("Date"))).getTime();
+ timestamp = String.valueOf(epoch);
+ } catch (ParseException pe) {
+ LOG.warn(pe.getMessage());
+ }
+ return timestamp;
+ } else {
+ return ifNullString(metadata.get("Date"));
+ }
+ }
+
+ protected String getResponseServer() {
+ return ifNullString(metadata.get("Server"));
+ }
+
+ protected String getResponseContent() {
+ return new String(content.getContent());
+ }
+
+ protected String getKey() {
+ if (this.reverseKey) {
+ return this.reverseKeyValue;
+ }
+ else {
+ return url;
+ }
+ }
+
+ protected String getImported() {
+ if (this.simpleDateFormat) {
+ String timestamp = null;
+ try {
+ long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z").parse(ifNullString(metadata.get("Date"))).getTime();
+ timestamp = String.valueOf(epoch);
+ } catch (ParseException pe) {
+ LOG.warn(pe.getMessage());
+ }
+ return timestamp;
+ } else {
+ return ifNullString(metadata.get("Date"));
+ }
+ }
+
+ private static String ifNullString(String value) {
+ return (value != null) ? value : "";
+ }
+
+ private void startHeaders(String key, boolean nested, boolean newline) throws IOException {
+ if (this.jsonArray) {
+ startArray(key, nested, newline);
+ }
+ else {
+ startObject(key);
+ }
+ }
+
+ private void closeHeaders(String key, boolean nested, boolean newline) throws IOException {
+ if (this.jsonArray) {
+ closeArray(key, nested, newline);
+ }
+ else {
+ closeObject(key);
+ }
+ }
+
+ private void writeKeyValueWrapper(String key, String value) throws IOException {
+ if (this.jsonArray) {
+ startArray(null, true, false);
+ writeArrayValue(key);
+ writeArrayValue(value);
+ closeArray(null, true, false);
+ }
+ else {
+ writeKeyValue(key, value);
+ }
+ }
+
+ @Override
+ public void close() {}
}