You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by le...@apache.org on 2016/08/23 03:16:34 UTC
[1/2] nutch git commit: NUTCH-2302 RAMConfManager Could Be
Constructed With Custom Configuration
Repository: nutch
Updated Branches:
refs/heads/2.x 5c3a38128 -> 22683a1df
NUTCH-2302 RAMConfManager Could Be Constructed With Custom Configuration
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/fd722c89
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/fd722c89
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/fd722c89
Branch: refs/heads/2.x
Commit: fd722c896468fe047758891d75a58259c88289d8
Parents: b7f3fce
Author: Furkan KAMACI <fu...@gmail.com>
Authored: Sat Aug 20 16:08:47 2016 +0300
Committer: Furkan KAMACI <fu...@gmail.com>
Committed: Sun Aug 21 00:40:45 2016 +0300
----------------------------------------------------------------------
.../apache/nutch/api/impl/RAMConfManager.java | 65 ++++++++++++++++++++
1 file changed, 65 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/fd722c89/src/java/org/apache/nutch/api/impl/RAMConfManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/api/impl/RAMConfManager.java b/src/java/org/apache/nutch/api/impl/RAMConfManager.java
index 2afd658..13c05fd 100644
--- a/src/java/org/apache/nutch/api/impl/RAMConfManager.java
+++ b/src/java/org/apache/nutch/api/impl/RAMConfManager.java
@@ -33,19 +33,44 @@ import org.apache.nutch.util.NutchConfiguration;
import com.google.common.collect.Maps;
+/**
+ * Configuration manager which holds a map of {@link Configuration} type configurations and ids.
+ */
public class RAMConfManager implements ConfManager {
private Map<String, Configuration> configurations = Maps.newConcurrentMap();
private AtomicInteger newConfigId = new AtomicInteger();
+ /**
+ * Public constructor which creates a default configuration with id of {@link ConfigResource#DEFAULT}.
+ */
public RAMConfManager() {
configurations.put(ConfigResource.DEFAULT, NutchConfiguration.create());
}
+ /**
+ * Public constructor which accepts a configuration id and {@link Configuration} type configuration.
+ */
+ public RAMConfManager(String confId, Configuration configuration) {
+ configurations.put(confId, configuration);
+ }
+
+ /**
+ * Lists configuration keys.
+ *
+ * @return Set of configuration keys
+ */
public Set<String> list() {
return configurations.keySet();
}
+ /**
+ * Returns configuration map for give configuration id.
+ *
+ * @param confId Configuration id.
+ * @return Configuration for given configuration id.
+ * {@link ConfigResource#DEFAULT} is used if given configuration id is null.
+ */
public Configuration get(String confId) {
if (confId == null) {
return configurations.get(ConfigResource.DEFAULT);
@@ -53,6 +78,13 @@ public class RAMConfManager implements ConfManager {
return configurations.get(confId);
}
+ /**
+ * Returns configuration map for give configuration id.
+ * An empty map is returned if a configuration could not be retrieved for given configuration id.
+ *
+ * @param confId Configuration id
+ * @return map of configurations
+ */
public Map<String, String> getAsMap(String confId) {
Configuration configuration = configurations.get(confId);
if (configuration == null) {
@@ -68,6 +100,13 @@ public class RAMConfManager implements ConfManager {
return configMap;
}
+ /**
+ * Sets a property for the configuration which has given configuration id.
+ *
+ * @param confId Configuration id
+ * @param propName property name to set
+ * @param propValue property value to set
+ */
public void setProperty(String confId, String propName, String propValue) {
if (!configurations.containsKey(confId)) {
throw new IllegalArgumentException("Unknown configId '" + confId + "'");
@@ -76,10 +115,23 @@ public class RAMConfManager implements ConfManager {
conf.set(propName, propValue);
}
+ /**
+ * Deletes configuration for given configuration id.
+ *
+ * @param confId Configuration id
+ */
public void delete(String confId) {
configurations.remove(confId);
}
+ /**
+ * Creates hadoop configuration for given Nutch configuration.
+ * Checks whether it can create a Nutch configuration or not before it creates.
+ * Throws {@link IllegalArgumentException} if can not pass {{@link #canCreate(NutchConfig)}}.
+ *
+ * @param nutchConfig Nutch configuration
+ * @return created configuration id
+ */
@Override
public String create(NutchConfig nutchConfig) {
if (StringUtils.isBlank(nutchConfig.getConfigId())) {
@@ -94,6 +146,14 @@ public class RAMConfManager implements ConfManager {
return nutchConfig.getConfigId();
}
+ /**
+ * Checks can create a Nutch configuration or not.
+ *
+ * @param nutchConfig Nutch configuration
+ * @return True if forcing is enabled at Nutch configuration.
+ * Otherwise makes a check based on whether there is an existing configuration at configuration set
+ * with same configuration id of given Nutch configuration.
+ */
private boolean canCreate(NutchConfig nutchConfig) {
if (nutchConfig.isForce()) {
return true;
@@ -104,6 +164,11 @@ public class RAMConfManager implements ConfManager {
return false;
}
+ /**
+ * Creates a Hadoop configuration from given Nutch configuration.
+ *
+ * @param nutchConfig Nutch configuration.
+ */
private void createHadoopConfig(NutchConfig nutchConfig) {
Configuration conf = NutchConfiguration.create();
configurations.put(nutchConfig.getConfigId(), conf);
[2/2] nutch git commit: Merge branch '2.x' of
https://git-wip-us.apache.org/repos/asf/nutch into 2.x
Posted by le...@apache.org.
Merge branch '2.x' of https://git-wip-us.apache.org/repos/asf/nutch into 2.x
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/22683a1d
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/22683a1d
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/22683a1d
Branch: refs/heads/2.x
Commit: 22683a1df83fb8100acdda388382e181c1b5b43d
Parents: fd722c8 5c3a381
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Mon Aug 22 20:23:55 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Mon Aug 22 20:23:55 2016 -0700
----------------------------------------------------------------------
conf/nutch-default.xml | 7 +++++++
src/java/org/apache/nutch/crawl/GeneratorMapper.java | 7 +++++++
2 files changed, 14 insertions(+)
----------------------------------------------------------------------