You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2010/08/26 16:55:00 UTC

svn commit: r989726 - in /nutch/trunk: CHANGES.txt src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java

Author: jnioche
Date: Thu Aug 26 14:54:59 2010
New Revision: 989726

URL: http://svn.apache.org/viewvc?rev=989726&view=rev
Log:
NUTCH-716 - subcollections

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java
    nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java

Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=989726&r1=989725&r2=989726&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Aug 26 14:54:59 2010
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 2.0 - Current Development
 
+* NUTCH-716 Make subcollection index filed multivalued (Dmitry Lihachev via jnioche)
+
 * NUTCH-884 FetcherJob should run more reduce tasks than default (ab)
 
 * NUTCH-883 Remove unused parameters from nutch-default.xml (jnioche)

Modified: nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java?rev=989726&r1=989725&r2=989726&view=diff
==============================================================================
--- nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java (original)
+++ nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java Thu Aug 26 14:54:59 2010
@@ -22,9 +22,12 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import org.slf4j.Logger;
@@ -43,196 +46,199 @@ import org.w3c.dom.NodeList;
 
 public class CollectionManager extends Configured {
 
-  public static final String DEFAULT_FILE_NAME = "subcollections.xml";
+	public static final String DEFAULT_FILE_NAME = "subcollections.xml";
 
-  static final Logger LOG = LoggerFactory.getLogger(CollectionManager.class);
+	static final Logger LOG = LoggerFactory.getLogger(CollectionManager.class);
 
-  transient Map collectionMap = new HashMap();
+	transient Map collectionMap = new HashMap();
 
-  transient URL configfile;
-  
-  public CollectionManager(Configuration conf) {
-    super(conf);
-    init();
-  }
-  
-  /** 
-   * Used for testing
-   */
-  protected CollectionManager(){
-    super(NutchConfiguration.create());
-  }
-
-  protected void init(){
-    try {
-      if (LOG.isInfoEnabled()) { LOG.info("initializing CollectionManager"); }
-      // initialize known subcollections
-      configfile = getConf().getResource(
-          getConf().get("subcollections.config", DEFAULT_FILE_NAME));
-
-      InputStream input = getConf().getConfResourceAsInputStream(
-          getConf().get("subcollections.config", DEFAULT_FILE_NAME));
-      parse(input);
-    } catch (Exception e) {
-      if (LOG.isWarnEnabled()) {
-        LOG.warn("Error occured:" + e);
-        e.printStackTrace(LogUtil.getWarnStream(LOG));
-      }
-    }
-  }
-
-  protected void parse(InputStream input) {
-    Element collections = DomUtil.getDom(input);
-
-    if (collections != null) {
-      NodeList nodeList = collections
-          .getElementsByTagName(Subcollection.TAG_COLLECTION);
-
-      if (LOG.isInfoEnabled()) {
-        LOG.info("file has" + nodeList.getLength() + " elements");
-      }
-      
-      for (int i = 0; i < nodeList.getLength(); i++) {
-        Element scElem = (Element) nodeList.item(i);
-        Subcollection subCol = new Subcollection(getConf());
-        subCol.initialize(scElem);
-        collectionMap.put(subCol.name, subCol);
-      }
-    } else if (LOG.isInfoEnabled()) {
-      LOG.info("Cannot find collections");
-    }
-  }
-  
-  public static CollectionManager getCollectionManager(Configuration conf) {
-    String key = "collectionmanager";
-    ObjectCache objectCache = ObjectCache.get(conf);
-    CollectionManager impl = (CollectionManager)objectCache.getObject(key);
-    if (impl == null) {
-      try {
-        if (LOG.isInfoEnabled()) {
-          LOG.info("Instantiating CollectionManager");
-        }
-        impl=new CollectionManager(conf);
-        objectCache.setObject(key,impl);
-      } catch (Exception e) {
-        throw new RuntimeException("Couldn't create CollectionManager",e);
-      }
-    }
-    return impl;
-  }
-
-  /**
-   * Returns named subcollection
-   * 
-   * @param id
-   * @return Named SubCollection (or null if not existing)
-   */
-  public Subcollection getSubColection(final String id) {
-    return (Subcollection) collectionMap.get(id);
-  }
-
-  /**
-   * Delete named subcollection
-   * 
-   * @param id
-   *          Id of SubCollection to delete
-   */
-  public void deleteSubCollection(final String id) throws IOException {
-    final Subcollection subCol = getSubColection(id);
-    if (subCol != null) {
-      collectionMap.remove(id);
-    }
-  }
-
-  /**
-   * Create a new subcollection.
-   * 
-   * @param name
-   *          Name of SubCollection to create
-   * @return Created SubCollection or null if allready existed
-   */
-  public Subcollection createSubCollection(final String id, final String name) {
-    Subcollection subCol = null;
-
-    if (!collectionMap.containsKey(id)) {
-      subCol = new Subcollection(id, name, getConf());
-      collectionMap.put(id, subCol);
-    }
-
-    return subCol;
-  }
-
-  /**
-   * Return names of collections url is part of
-   * 
-   * @param url
-   *          The url to test against Collections
-   * @return Space delimited string of collection names url is part of
-   */
-  public String getSubCollections(final String url) {
-    StringBuilder collections = new StringBuilder();
-    final Iterator iterator = collectionMap.values().iterator();
-
-    while (iterator.hasNext()) {
-      final Subcollection subCol = (Subcollection) iterator.next();
-      if (subCol.filter(url) != null) {
-        if (collections.length() > 0) {
-          collections.append(' ');
-        }
-        collections.append(subCol.name);
-      }
-    }
-    if (LOG.isTraceEnabled()) { LOG.trace("subcollections:" + collections); }
-    
-    return collections.toString();
-  }
-
-  /**
-   * Returns all collections
-   * 
-   * @return All collections CollectionManager knows about
-   */
-  public Collection getAll() {
-    return collectionMap.values();
-  }
-
-  /**
-   * Save collections into file
-   * 
-   * @throws Exception
-   */
-  public void save() throws IOException {
-    try {
-      final FileOutputStream fos = new FileOutputStream(new File(configfile
-          .getFile()));
-      final Document doc = new DocumentImpl();
-      final Element collections = doc
-          .createElement(Subcollection.TAG_COLLECTIONS);
-      final Iterator iterator = collectionMap.values().iterator();
-
-      while (iterator.hasNext()) {
-        final Subcollection subCol = (Subcollection) iterator.next();
-        final Element collection = doc
-            .createElement(Subcollection.TAG_COLLECTION);
-        collections.appendChild(collection);
-        final Element name = doc.createElement(Subcollection.TAG_NAME);
-        name.setNodeValue(subCol.getName());
-        collection.appendChild(name);
-        final Element whiteList = doc
-            .createElement(Subcollection.TAG_WHITELIST);
-        whiteList.setNodeValue(subCol.getWhiteListString());
-        collection.appendChild(whiteList);
-        final Element blackList = doc
-            .createElement(Subcollection.TAG_BLACKLIST);
-        blackList.setNodeValue(subCol.getBlackListString());
-        collection.appendChild(blackList);
-      }
-
-      DomUtil.saveDom(fos, collections);
-      fos.flush();
-      fos.close();
-    } catch (FileNotFoundException e) {
-      throw new IOException(e.toString());
-    }
-  }
+	transient URL configfile;
+
+	public CollectionManager(Configuration conf) {
+		super(conf);
+		init();
+	}
+
+	/**
+	 * Used for testing
+	 */
+	protected CollectionManager() {
+		super(NutchConfiguration.create());
+	}
+
+	protected void init() {
+		try {
+			if (LOG.isInfoEnabled()) {
+				LOG.info("initializing CollectionManager");
+			}
+			// initialize known subcollections
+			configfile = getConf().getResource(
+					getConf().get("subcollections.config", DEFAULT_FILE_NAME));
+
+			InputStream input = getConf().getConfResourceAsInputStream(
+					getConf().get("subcollections.config", DEFAULT_FILE_NAME));
+			parse(input);
+		} catch (Exception e) {
+			if (LOG.isWarnEnabled()) {
+				LOG.warn("Error occured:" + e);
+				e.printStackTrace(LogUtil.getWarnStream(LOG));
+			}
+		}
+	}
+
+	protected void parse(InputStream input) {
+		Element collections = DomUtil.getDom(input);
+
+		if (collections != null) {
+			NodeList nodeList = collections
+					.getElementsByTagName(Subcollection.TAG_COLLECTION);
+
+			if (LOG.isInfoEnabled()) {
+				LOG.info("file has" + nodeList.getLength() + " elements");
+			}
+
+			for (int i = 0; i < nodeList.getLength(); i++) {
+				Element scElem = (Element) nodeList.item(i);
+				Subcollection subCol = new Subcollection(getConf());
+				subCol.initialize(scElem);
+				collectionMap.put(subCol.name, subCol);
+			}
+		} else if (LOG.isInfoEnabled()) {
+			LOG.info("Cannot find collections");
+		}
+	}
+
+	public static CollectionManager getCollectionManager(Configuration conf) {
+		String key = "collectionmanager";
+		ObjectCache objectCache = ObjectCache.get(conf);
+		CollectionManager impl = (CollectionManager) objectCache.getObject(key);
+		if (impl == null) {
+			try {
+				if (LOG.isInfoEnabled()) {
+					LOG.info("Instantiating CollectionManager");
+				}
+				impl = new CollectionManager(conf);
+				objectCache.setObject(key, impl);
+			} catch (Exception e) {
+				throw new RuntimeException("Couldn't create CollectionManager",
+						e);
+			}
+		}
+		return impl;
+	}
+
+	/**
+	 * Returns named subcollection
+	 * 
+	 * @param id
+	 * @return Named SubCollection (or null if not existing)
+	 */
+	public Subcollection getSubColection(final String id) {
+		return (Subcollection) collectionMap.get(id);
+	}
+
+	/**
+	 * Delete named subcollection
+	 * 
+	 * @param id
+	 *            Id of SubCollection to delete
+	 */
+	public void deleteSubCollection(final String id) throws IOException {
+		final Subcollection subCol = getSubColection(id);
+		if (subCol != null) {
+			collectionMap.remove(id);
+		}
+	}
+
+	/**
+	 * Create a new subcollection.
+	 * 
+	 * @param name
+	 *            Name of SubCollection to create
+	 * @return Created SubCollection or null if allready existed
+	 */
+	public Subcollection createSubCollection(final String id, final String name) {
+		Subcollection subCol = null;
+
+		if (!collectionMap.containsKey(id)) {
+			subCol = new Subcollection(id, name, getConf());
+			collectionMap.put(id, subCol);
+		}
+
+		return subCol;
+	}
+
+	/**
+	 * Return names of collections url is part of
+	 * 
+	 * @param url
+	 *            The url to test against Collections
+	 * @return Space delimited string of collection names url is part of
+	 */
+	public List<String> getSubCollections(final String url) {
+		List<String> collections = new ArrayList<String>();
+		final Iterator iterator = collectionMap.values().iterator();
+
+		while (iterator.hasNext()) {
+			final Subcollection subCol = (Subcollection) iterator.next();
+			if (subCol.filter(url) != null) {
+				collections.add(subCol.name);
+			}
+		}
+		if (LOG.isTraceEnabled()) {
+			LOG.trace("subcollections:"
+					+ Arrays.toString(collections.toArray()));
+		}
+
+		return collections;
+	}
+
+	/**
+	 * Returns all collections
+	 * 
+	 * @return All collections CollectionManager knows about
+	 */
+	public Collection getAll() {
+		return collectionMap.values();
+	}
+
+	/**
+	 * Save collections into file
+	 * 
+	 * @throws Exception
+	 */
+	public void save() throws IOException {
+		try {
+			final FileOutputStream fos = new FileOutputStream(new File(
+					configfile.getFile()));
+			final Document doc = new DocumentImpl();
+			final Element collections = doc
+					.createElement(Subcollection.TAG_COLLECTIONS);
+			final Iterator iterator = collectionMap.values().iterator();
+
+			while (iterator.hasNext()) {
+				final Subcollection subCol = (Subcollection) iterator.next();
+				final Element collection = doc
+						.createElement(Subcollection.TAG_COLLECTION);
+				collections.appendChild(collection);
+				final Element name = doc.createElement(Subcollection.TAG_NAME);
+				name.setNodeValue(subCol.getName());
+				collection.appendChild(name);
+				final Element whiteList = doc
+						.createElement(Subcollection.TAG_WHITELIST);
+				whiteList.setNodeValue(subCol.getWhiteListString());
+				collection.appendChild(whiteList);
+				final Element blackList = doc
+						.createElement(Subcollection.TAG_BLACKLIST);
+				blackList.setNodeValue(subCol.getBlackListString());
+				collection.appendChild(blackList);
+			}
+
+			DomUtil.saveDom(fos, collections);
+			fos.flush();
+			fos.close();
+		} catch (FileNotFoundException e) {
+			throw new IOException(e.toString());
+		}
+	}
 }

Modified: nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java?rev=989726&r1=989725&r2=989726&view=diff
==============================================================================
--- nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java (original)
+++ nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java Thu Aug 26 14:54:59 2010
@@ -60,9 +60,9 @@ public class SubcollectionIndexingFilter
 	 * @param url
 	 */
 	private void addSubCollectionField(NutchDocument doc, String url) {
-		String collname = CollectionManager.getCollectionManager(getConf())
-				.getSubCollections(url);
-		doc.add(FIELD_NAME, collname);
+		for (String collname: CollectionManager.getCollectionManager(getConf()).getSubCollections(url)) {
+			doc.add(FIELD_NAME, collname);
+		}
 	}
 
 	@Override