You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2018/03/08 13:03:31 UTC
[nutch] branch master updated: NUTCH-2411 Index-metadata to support
indexing multiple values for a field
This is an automated email from the ASF dual-hosted git repository.
markus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 9a77f43 NUTCH-2411 Index-metadata to support indexing multiple values for a field
9a77f43 is described below
commit 9a77f43774b2c3cd70785895afb989e9ee2d8d5f
Author: Markus Jelsma <ma...@apache.org>
AuthorDate: Thu Mar 8 14:03:12 2018 +0100
NUTCH-2411 Index-metadata to support indexing multiple values for a field
---
conf/nutch-default.xml | 9 ++++++
.../nutch/indexer/metadata/MetadataIndexer.java | 35 ++++++++++++++++++----
2 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 87c4058..71ef51b 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -1738,6 +1738,15 @@ visit https://wiki.apache.org/nutch/SimilarityScoringFilter-->
</description>
</property>
+<property>
+ <name>index.metadata.separator</name>
+ <value></value>
+ <description>
+ Separator to use if you want to index multiple values for a given field. Leave empty to
+ treat each value as a single value.
+ </description>
+</property>
+
<!-- index-geoip plugin properties -->
<property>
<name>index.geoip.usage</name>
diff --git a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
index edb8b15..74d9eb1 100644
--- a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
+++ b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
@@ -17,9 +17,12 @@
package org.apache.nutch.indexer.metadata;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -35,17 +38,21 @@ import org.apache.nutch.parse.Parse;
* Indexer which can be configured to extract metadata from the crawldb, parse
* metadata or content metadata. You can specify the properties "index.db.md",
* "index.parse.md" or "index.content.md" who's values are comma-delimited
- * Example : key1,key2,key3.
+ * <value>key1,key2,key3</value>.
*/
public class MetadataIndexer implements IndexingFilter {
private Configuration conf;
private String[] dbFieldnames;
private Map<String, String> parseFieldnames;
private String[] contentFieldnames;
+ private String separator;
+ private Set<String> mvFields;
private static final String db_CONF_PROPERTY = "index.db.md";
private static final String parse_CONF_PROPERTY = "index.parse.md";
private static final String content_CONF_PROPERTY = "index.content.md";
-
+ private static final String separator_CONF_PROPERTY = "index.metadata.separator";
+ private static final String mvfields_CONF_PROPERTY = "index.metadata.multivalued.fields";
+
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
@@ -58,7 +65,7 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : dbFieldnames) {
Writable metadata = datum.getMetaData().get(new Text(metatag));
if (metadata != null)
- doc.add(metatag, metadata.toString());
+ add(doc, metatag, metadata.toString());
}
}
@@ -67,7 +74,7 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : parseFieldnames.keySet()) {
for (String value : parse.getData().getParseMeta().getValues(metatag)) {
if (value != null)
- doc.add(parseFieldnames.get(metatag), value);
+ add(doc, parseFieldnames.get(metatag), value);
}
}
}
@@ -77,13 +84,27 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : contentFieldnames) {
for (String value : parse.getData().getContentMeta().getValues(metatag)) {
if (value != null)
- doc.add(metatag, value);
+ add(doc, metatag, value);
}
}
}
return doc;
}
+
+ protected void add(NutchDocument doc, String key, String value) {
+ if (separator == null || value.indexOf(separator) == -1 || !mvFields.contains(key)) {
+ doc.add(key, value);
+ } else {
+ String[] parts = value.split(separator);
+ for (String part : parts) {
+ part = part.trim();
+ if (part.length() != 0) {
+ doc.add(key, part);
+ }
+ }
+ }
+ }
public void setConf(Configuration conf) {
this.conf = conf;
@@ -93,7 +114,9 @@ public class MetadataIndexer implements IndexingFilter {
parseFieldnames.put(metatag.toLowerCase(Locale.ROOT), metatag);
}
contentFieldnames = conf.getStrings(content_CONF_PROPERTY);
-
+
+ separator = conf.get(separator_CONF_PROPERTY, null);
+ mvFields = new HashSet(Arrays.asList(conf.getStrings(mvfields_CONF_PROPERTY, new String[0])));
// TODO check conflict between field names e.g. could have same label
// from different sources
--
To stop receiving notification emails like this one, please contact
markus@apache.org.