You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by le...@apache.org on 2026/04/14 02:18:57 UTC
(nutch) branch master updated: NUTCH-3165 Remove the Nutch web service (#908)
This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 3b663de2e NUTCH-3165 Remove the Nutch web service (#908)
3b663de2e is described below
commit 3b663de2e4c829267297814db8d231d037c586f7
Author: Lewis John McGibbney <le...@apache.org>
AuthorDate: Mon Apr 13 19:18:51 2026 -0700
NUTCH-3165 Remove the Nutch web service (#908)
---
.gitignore | 2 +
build.xml | 24 ---
docker/Dockerfile | 68 +-----
docker/README.md | 52 +----
docker/config/supervisord_startserver.conf | 47 -----
docker/config/supervisord_startserver_webapp.conf | 69 ------
ivy/ivy.xml | 10 +-
src/bin/nutch | 5 -
src/java/org/apache/nutch/crawl/CrawlDb.java | 4 +-
src/java/org/apache/nutch/crawl/Injector.java | 11 +-
src/java/org/apache/nutch/crawl/LinkDb.java | 4 +-
src/java/org/apache/nutch/fetcher/FetchNode.java | 72 -------
src/java/org/apache/nutch/fetcher/FetchNodeDb.java | 48 -----
.../org/apache/nutch/fetcher/FetcherThread.java | 29 ---
src/java/org/apache/nutch/indexer/IndexingJob.java | 4 +-
src/java/org/apache/nutch/metadata/Nutch.java | 21 +-
src/java/org/apache/nutch/parse/ParseSegment.java | 4 +-
src/java/org/apache/nutch/service/ConfManager.java | 38 ----
src/java/org/apache/nutch/service/JobManager.java | 43 ----
src/java/org/apache/nutch/service/NutchReader.java | 38 ----
src/java/org/apache/nutch/service/NutchServer.java | 234 ---------------------
src/java/org/apache/nutch/service/SeedManager.java | 32 ---
.../apache/nutch/service/impl/ConfManagerImpl.java | 138 ------------
.../org/apache/nutch/service/impl/JobFactory.java | 74 -------
.../apache/nutch/service/impl/JobManagerImpl.java | 95 ---------
.../org/apache/nutch/service/impl/JobWorker.java | 117 -----------
.../org/apache/nutch/service/impl/LinkReader.java | 171 ---------------
.../org/apache/nutch/service/impl/NodeReader.java | 179 ----------------
.../service/impl/NutchServerPoolExecutor.java | 130 ------------
.../apache/nutch/service/impl/SeedManagerImpl.java | 61 ------
.../apache/nutch/service/impl/SequenceReader.java | 171 ---------------
.../apache/nutch/service/impl/ServiceWorker.java | 50 -----
.../nutch/service/model/request/DbQuery.java | 56 -----
.../nutch/service/model/request/JobConfig.java | 73 -------
.../nutch/service/model/request/NutchConfig.java | 51 -----
.../nutch/service/model/request/ReaderConfig.java | 30 ---
.../nutch/service/model/request/SeedList.java | 103 ---------
.../nutch/service/model/request/SeedUrl.java | 89 --------
.../nutch/service/model/request/ServiceConfig.java | 51 -----
.../service/model/response/FetchNodeDbInfo.java | 107 ----------
.../nutch/service/model/response/JobInfo.java | 100 ---------
.../service/model/response/NutchServerInfo.java | 55 -----
.../nutch/service/model/response/ServiceInfo.java | 32 ---
.../nutch/service/resources/AbstractResource.java | 45 ----
.../nutch/service/resources/AdminResource.java | 87 --------
.../nutch/service/resources/ConfigResource.java | 132 ------------
.../apache/nutch/service/resources/DbResource.java | 145 -------------
.../nutch/service/resources/JobResource.java | 99 ---------
.../nutch/service/resources/ReaderResouce.java | 177 ----------------
.../nutch/service/resources/SeedResource.java | 113 ----------
.../nutch/service/resources/ServicesResource.java | 83 --------
.../apache/nutch/tools/CommonCrawlDataDumper.java | 2 +-
.../org/apache/nutch/crawl/CrawlDBTestUtil.java | 73 ++++---
src/test/org/apache/nutch/fetcher/TestFetcher.java | 16 +-
.../org/apache/nutch/service/TestNutchServer.java | 66 ------
55 files changed, 90 insertions(+), 3740 deletions(-)
diff --git a/.gitignore b/.gitignore
index fd642ad01..b7dbff406 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,5 @@ lib/native/libhadooputils.*
lib/native/libhdfs.*
lib/native/libhdfspp.*
lib/native/libnativetask.*
+#Jacoco download
+ivy/jacoco*
diff --git a/build.xml b/build.xml
index 2dffdb769..08ee22f85 100644
--- a/build.xml
+++ b/build.xml
@@ -382,30 +382,6 @@
</artifact:mvn>
</target>
- <!-- ================================================================== -->
- <!-- Generate REST API Documentation with Miredot -->
- <!-- ================================================================== -->
- <target name="restdocs" depends="makepom" description="--> generate REST API Documentation with Miredot">
-
- <artifact:mvn mavenHome="${env.MVN_HOME}" fork="true" failonerror="true">
- <jvmarg value="-Dmaven.multiModuleProjectDirectory=false" />
- <arg value="package"/>
- <arg value="-DskipTests"/>
- <arg value="-e"/>
- <!--arg value="-o"/-->
- <!-- run offline (-o): must not download dependencies as this is
- done from http://repo1.maven.org/ hardwired in
- maven-ant-tasks-2.1.3.jar, see NUTCH-2722.
-
- Dependencies and plugins need to be resolved and cached locally beforehand
- by running
- `mvn dependency:resolve`
- resp.
- `mvn dependency:resolve-plugins`
- after the pom.xml has been generated. -->
- </artifact:mvn>
- </target>
-
<!-- ================================================================== -->
<!-- Make job jar -->
<!-- ================================================================== -->
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 93985f228..392dfdb86 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -16,17 +16,7 @@
# NOTE TO DEVELOPERS: Make sure this file passes linting tests
# by running https://github.com/replicatedhq/dockerfilelint
-# BUILD_MODE can be either
-# 0 == Nutch master branch source install with 'crawl' and 'nutch' scripts on PATH
-# 1 == Same as mode 0 with addition of Nutch REST Server
-# 2 == Same as mode 1 with addition of Nutch WebApp
-ARG BUILD_MODE=0
-
-FROM alpine:3.19 AS base
-
-ARG SERVER_PORT=8081
-ARG SERVER_HOST=0.0.0.0
-ARG WEBAPP_PORT=8080
+FROM alpine:3.19
LABEL maintainer="Apache Nutch Developers <de...@nutch.apache.org>"
LABEL org.opencontainers.image.authors="Apache Nutch Developers <de...@nutch.apache.org>"
@@ -42,7 +32,7 @@ WORKDIR /root/
# Install dependencies
RUN apk update
-RUN apk --no-cache add apache-ant bash git openjdk17 supervisor
+RUN apk --no-cache add apache-ant bash git openjdk17
# Establish environment variables
RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.bashrc
@@ -61,56 +51,6 @@ RUN git clone https://github.com/apache/nutch.git nutch_source && \
RUN ln -sf $NUTCH_HOME/bin/nutch /usr/local/bin/
RUN ln -sf $NUTCH_HOME/bin/crawl /usr/local/bin/
-FROM base AS branch-version-0
-
-RUN echo "Nutch master branch source install with 'crawl' and 'nutch' scripts on PATH"
-
-FROM base AS branch-version-1
-
-RUN echo "Nutch master branch source install with 'crawl' and 'nutch' scripts on PATH and Nutch REST Server on $SERVER_HOST:$SERVER_PORT"
-ARG SERVER_PORT=8081
-ARG SERVER_HOST=0.0.0.0
-
-ENV SERVER_PORT=$SERVER_PORT
-ENV SERVER_HOST=$SERVER_HOST
-
-# Arrange necessary setup for supervisord
-RUN mkdir -p /var/log/supervisord
-COPY ./config/supervisord_startserver.conf /etc/supervisord.conf
-
-# Expose port for server which can only be accessed if
-# the same port is published when the container is run.
-EXPOSE $SERVER_PORT
-
-ENTRYPOINT [ "supervisord", "--nodaemon", "--configuration", "/etc/supervisord.conf" ]
-
-FROM base AS branch-version-2
-
-RUN echo "Nutch master branch source install with 'crawl' and 'nutch' scripts on PATH, Nutch REST Server on $SERVER_HOST:$SERVER_PORT and WebApp on this container port $WEBAPP_PORT"
-ARG SERVER_PORT=8081
-ARG SERVER_HOST=0.0.0.0
-ARG WEBAPP_PORT=8080
-
-ENV SERVER_PORT=$SERVER_PORT
-ENV SERVER_HOST=$SERVER_HOST
-ENV WEBAPP_PORT=$WEBAPP_PORT
-
-# Install the webapp
-RUN apk --no-cache add maven
-RUN git clone https://github.com/apache/nutch-webapp.git nutch_webapp && \
- cd nutch_webapp && \
- mvn package
-
-# Arrange necessary setup for supervisord
-RUN mkdir -p /var/log/supervisord
-COPY ./config/supervisord_startserver_webapp.conf /etc/supervisord.conf
-
-# Expose ports for server and webapp, these can only be accessed if
-# the same ports are published when the container is run.
-EXPOSE $SERVER_PORT
-EXPOSE $WEBAPP_PORT
-
-ENTRYPOINT [ "supervisord", "--nodaemon", "--configuration", "/etc/supervisord.conf" ]
-
-FROM branch-version-$BUILD_MODE AS final
RUN echo "Successfully built image, see https://s.apache.org/m5933 for guidance on running a container instance."
+
+CMD ["/bin/bash"]
diff --git a/docker/README.md b/docker/README.md
index 720fdf816..6c6304481 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -44,16 +44,9 @@ The easiest way to do this:
2. Build from files in this directory:
-There are three build **modes** which can be activated using the `--build-arg BUILD_MODE=0` flag. All values used here are defaults.
- * 0 == Nutch master branch source install with `crawl` and `nutch` scripts on `$PATH`
- * 1 == Same as mode 0 with addition of **Nutch REST Server**; additional build args `--build-arg SERVER_PORT=8081` and `--build-arg SERVER_HOST=0.0.0.0`
- * 2 == Same as mode 1 with addition of **Nutch WebApp**; additional build args `--build-arg WEBAPP_PORT=8080`
-
-For example, if you wanted to install Nutch master branch and run both the Nutch REST server and webapp then run the following
-
```bash
$(boot2docker shellinit | grep export) #may not be necessary
-docker build -t apache/nutch . --build-arg BUILD_MODE=2 --build-arg SERVER_PORT=8081 --build-arg SERVER_HOST=0.0.0.0 --build-arg WEBAPP_PORT=8080
+docker build -t apache/nutch .
```
## Security and plugin directories
@@ -70,52 +63,19 @@ boot2docker up
$(boot2docker shellinit | grep export)
```
-Run a container
-
-```bash
-docker run -t -i -d -p 8080:8080 -p 8081:8081 --name nutchcontainer apache/nutch
-c5401810e50a606f43256b4b24602443508bd9badcf2b7493bd97839834571fc
-
-docker logs c5401810e50a606f43256b4b24602443508bd9badcf2b7493bd97839834571fc
-2021-06-29 19:14:32,922 CRIT Supervisor is running as root. Privileges were not dropped because no user is specified in the config file. If you intend to run as root, you can set user=root in the config file to avoid this message.
-2021-06-29 19:14:32,925 INFO supervisord started with pid 1
-2021-06-29 19:14:33,929 INFO spawned: 'nutchserver' with pid 8
-2021-06-29 19:14:33,932 INFO spawned: 'nutchwebapp' with pid 9
-2021-06-29 19:14:36,012 INFO success: nutchserver entered RUNNING state, process has stayed up for > than 2 seconds (startsecs)
-2021-06-29 19:14:36,012 INFO success: nutchwebapp entered RUNNING state, process has stayed up for > than 2 seconds (startsecs)
-```
-
-You can now access the webapp at `http://localhost:8080` and you can interact with the REST API e.g.
-
-```bash
-curl http://localhost:8080/admin
-{"startDate":1625118207995,"configuration":["default"],"jobs":[],"runningJobs":[]}
-```
-
-Attach to the container
+Run a container interactively (`nutch` and `crawl` are on `PATH`; default command is `bash`):
```bash
-docker exec -it c5401810e50a606f43256b4b24602443508bd9badcf2b7493bd97839834571fc /bin/bash
+docker run -t -i --name nutchcontainer apache/nutch
```
-View supervisord logs
-```bash
-cat /tmp/supervisord.log
-2021-06-29 19:14:32,922 CRIT Supervisor is running as root. Privileges were not dropped because no user is specified in the config file. If you intend to run as root, you can set user=root in the config file to avoid this message.
-2021-06-29 19:14:32,925 INFO supervisord started with pid 1
-2021-06-29 19:14:33,929 INFO spawned: 'nutchserver' with pid 8
-2021-06-29 19:14:33,932 INFO spawned: 'nutchwebapp' with pid 9
-2021-06-29 19:14:36,012 INFO success: nutchserver entered RUNNING state, process has stayed up for > than 2 seconds (startsecs)
-2021-06-29 19:14:36,012 INFO success: nutchwebapp entered RUNNING state, process has stayed up for > than 2 seconds (startsecs)
-```
-
-View supervisord subprocess logs
+In another terminal, attach to a running container if needed:
```bash
-ls /var/log/supervisord/
-nutchserver_stderr.log nutchserver_stdout.log nutchwebapp_stderr.log nutchwebapp_stdout.log
+docker exec -it nutchcontainer /bin/bash
```
Nutch is located in `$NUTCH_HOME` and is almost ready to run.
You will need to set seed URLs and update the `http.agent.name` configuration property in `$NUTCH_HOME/conf/nutch-site.xml` with your crawler's Agent Name.
For additional "getting started" information checkout the [Nutch Tutorial](https://cwiki.apache.org/confluence/display/NUTCH/NutchTutorial).
+
diff --git a/docker/config/supervisord_startserver.conf b/docker/config/supervisord_startserver.conf
deleted file mode 100644
index 18ecc948d..000000000
--- a/docker/config/supervisord_startserver.conf
+++ /dev/null
@@ -1,47 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[supervisord]
-childlogdir=/var/log/supervisord/
-logfile=/var/log/supervisord/supervisord.log ; (main log file;default $CWD/supervisord.log)
-logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)
-logfile_backups=10 ; (num of main logfile rotation backups;default 10)
-loglevel=info ; (log level;default info; others: debug,warn,trace)
-minfds=1024 ; (min. avail startup file descriptors;default 1024)
-minprocs=200 ; (min. avail process descriptors;default 200)
-nodaemon=false ; (start in foreground if true;default false)
-pidfile=/var/log/supervisord/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-user=root
-
-[program:nutchserver]
-autorestart=true
-autostart=true
-command=nutch startserver -port %(ENV_SERVER_PORT)s -host %(ENV_SERVER_HOST)s
-environment=PATH=/usr/local/bin:%(ENV_PATH)s
-process_name=%(program_name)s
-numprocs=1
-#redirect_stderr=true
-startsecs=0
-stderr_capture_maxbytes=10MB
-stderr_logfile=/var/log/supervisord/%(program_name)s_stderr.log
-stderr_logfile_backups=5
-stderr_logfile_maxbytes=10MB
-#stderr_syslog=
-stdout_capture_maxbytes=10MB
-stdout_logfile=/var/log/supervisord/%(program_name)s_stdout.log
-stdout_logfile_backups=5
-stdout_logfile_maxbytes=10MB
-#stdout_syslog=
-user=root
diff --git a/docker/config/supervisord_startserver_webapp.conf b/docker/config/supervisord_startserver_webapp.conf
deleted file mode 100644
index 25f653cda..000000000
--- a/docker/config/supervisord_startserver_webapp.conf
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[supervisord]
-childlogdir=/var/log/supervisord/
-logfile=/var/log/supervisord/supervisord.log ; (main log file;default $CWD/supervisord.log)
-logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)
-logfile_backups=10 ; (num of main logfile rotation backups;default 10)
-loglevel=info ; (log level;default info; others: debug,warn,trace)
-minfds=1024 ; (min. avail startup file descriptors;default 1024)
-minprocs=200 ; (min. avail process descriptors;default 200)
-nodaemon=false ; (start in foreground if true;default false)
-pidfile=/var/log/supervisord/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-user=root
-
-[program:nutchserver]
-autorestart=true
-autostart=true
-command=nutch startserver -port %(ENV_SERVER_PORT)s -host %(ENV_SERVER_HOST)s
-environment=PATH=/usr/local/bin:%(ENV_PATH)s
-process_name=%(program_name)s
-numprocs=1
-#redirect_stderr=true
-startsecs=0
-stderr_capture_maxbytes=10MB
-stderr_logfile=/var/log/supervisord/%(program_name)s_stderr.log
-stderr_logfile_backups=5
-stderr_logfile_maxbytes=10MB
-#stderr_syslog=
-stdout_capture_maxbytes=10MB
-stdout_logfile=/var/log/supervisord/%(program_name)s_stdout.log
-stdout_logfile_backups=5
-stdout_logfile_maxbytes=10MB
-#stdout_syslog=
-user=root
-
-[program:nutchwebapp]
-autorestart=true
-autostart=true
-directory=/root/nutch_webapp
-command=mvn jetty:run -Djetty.port=%(ENV_WEBAPP_PORT)s
-environment=PATH=/usr/local/bin:%(ENV_PATH)s
-process_name=%(program_name)s
-numprocs=1
-#redirect_stderr=true
-startsecs=0
-stderr_capture_maxbytes=10MB
-stderr_logfile=/var/log/supervisord/%(program_name)s_stderr.log
-stderr_logfile_backups=5
-stderr_logfile_maxbytes=10MB
-#stderr_syslog=
-stdout_capture_maxbytes=10MB
-stdout_logfile=/var/log/supervisord/%(program_name)s_stdout.log
-stdout_logfile_backups=5
-stdout_logfile_maxbytes=10MB
-#stdout_syslog=
-user=root
\ No newline at end of file
diff --git a/ivy/ivy.xml b/ivy/ivy.xml
index b6b8f67a9..7b32242cc 100644
--- a/ivy/ivy.xml
+++ b/ivy/ivy.xml
@@ -90,15 +90,9 @@
<exclude module="hadoop-client" />
</dependency>
- <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxws" rev="3.6.9" conf="*->default" />
- <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxrs" rev="3.6.9" conf="*->default" />
- <dependency org="org.apache.cxf" name="cxf-rt-transports-http" rev="3.6.9" conf="*->default" />
- <dependency org="org.apache.cxf" name="cxf-rt-transports-http-jetty" rev="3.6.9" conf="*->default" />
- <dependency org="org.apache.cxf" name="cxf-rt-rs-client" rev="3.6.9" conf="test->default" />
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.18.5" conf="*->default" />
<dependency org="com.fasterxml.jackson.core" name="jackson-annotations" rev="2.18.5" conf="*->default" />
<dependency org="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" rev="2.18.5" conf="*->default" />
- <dependency org="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-json-provider" rev="2.18.5" conf="*->default" />
<!-- WARC artifacts needed -->
<dependency org="org.netpreserve.commons" name="webarchive-commons" rev="3.0.2" conf="*->default">
@@ -132,8 +126,8 @@
<dependency org="org.testcontainers" name="testcontainers" rev="2.0.3" conf="test->default"/>
<dependency org="org.testcontainers" name="junit-jupiter" rev="1.21.4" conf="test->default"/>
- <!-- Jetty used to serve test pages for unit tests, but is also provided as dependency of Hadoop -->
- <dependency org="org.eclipse.jetty" name="jetty-server" rev="12.1.5" conf="test->default">
+ <!-- MockServer (https://www.mock-server.com/) for static HTTP content in unit tests. -->
+ <dependency org="org.mock-server" name="mockserver-netty" rev="5.15.0" conf="test->default">
<exclude org="ch.qos.reload4j" module="*" />
<exclude org="org.slf4j" module="slf4j-reload" />
</dependency>
diff --git a/src/bin/nutch b/src/bin/nutch
index d4b873cd0..87018ec32 100755
--- a/src/bin/nutch
+++ b/src/bin/nutch
@@ -119,9 +119,6 @@ if [ $# = 0 ]; then
echo " commoncrawldump exports crawled data from segments into common crawl data format encoded as CBOR"
echo " warc exports crawled data from segments at the WARC format"
echo ""
- echo " (Nutch Server)"
- echo " startserver runs the Nutch Server on localhost:8081"
- echo ""
echo " (or)"
echo " CLASSNAME run the main of the class named CLASSNAME"
echo ""
@@ -316,8 +313,6 @@ elif [ "$COMMAND" = "junit" ] ; then
done
fi
CLASS=org.junit.runner.JUnitCore
-elif [ "$COMMAND" = "startserver" ] ; then
- CLASS=org.apache.nutch.service.NutchServer
elif [ "$COMMAND" = "warc" ] ; then
CLASS=org.apache.nutch.tools.warc.WARCExporter
elif [ "$COMMAND" = "updatehostdb" ] ; then
diff --git a/src/java/org/apache/nutch/crawl/CrawlDb.java b/src/java/org/apache/nutch/crawl/CrawlDb.java
index 32081e1d6..386a1b5c9 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDb.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDb.java
@@ -281,8 +281,8 @@ public class CrawlDb extends NutchTool implements Tool {
}
}
- /*
- * Used for Nutch REST service
+ /**
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
*/
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId) throws Exception {
diff --git a/src/java/org/apache/nutch/crawl/Injector.java b/src/java/org/apache/nutch/crawl/Injector.java
index 3fe4ce9ce..ae5218369 100644
--- a/src/java/org/apache/nutch/crawl/Injector.java
+++ b/src/java/org/apache/nutch/crawl/Injector.java
@@ -43,7 +43,6 @@ import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
-import org.apache.nutch.service.NutchServer;
import org.apache.nutch.util.LockUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -638,25 +637,21 @@ public class Injector extends NutchTool implements Tool {
}
/**
- * Used by the Nutch REST service
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
*/
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId)
throws Exception {
if(args.size()<1){
- throw new IllegalArgumentException("Required arguments <url_dir> or <seedName>");
+ throw new IllegalArgumentException("Required argument " + Nutch.ARG_SEEDDIR);
}
Path input;
Object path = null;
if(args.containsKey(Nutch.ARG_SEEDDIR)) {
path = args.get(Nutch.ARG_SEEDDIR);
}
- else if(args.containsKey(Nutch.ARG_SEEDNAME)) {
- path = NutchServer.getInstance().getSeedManager().
- getSeedList((String)args.get(Nutch.ARG_SEEDNAME)).getSeedFilePath();
- }
else {
- throw new IllegalArgumentException("Required arguments <url_dir> or <seedName>");
+ throw new IllegalArgumentException("Required argument " + Nutch.ARG_SEEDDIR);
}
if(path instanceof Path) {
input = (Path) path;
diff --git a/src/java/org/apache/nutch/crawl/LinkDb.java b/src/java/org/apache/nutch/crawl/LinkDb.java
index 2f4a0dda4..e5ace0ae6 100644
--- a/src/java/org/apache/nutch/crawl/LinkDb.java
+++ b/src/java/org/apache/nutch/crawl/LinkDb.java
@@ -371,8 +371,8 @@ public class LinkDb extends NutchTool implements Tool {
}
}
- /*
- * Used for Nutch REST service
+ /**
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
*/
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId) throws Exception {
diff --git a/src/java/org/apache/nutch/fetcher/FetchNode.java b/src/java/org/apache/nutch/fetcher/FetchNode.java
deleted file mode 100644
index 837c6515d..000000000
--- a/src/java/org/apache/nutch/fetcher/FetchNode.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.fetcher;
-
-import java.net.MalformedURLException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.parse.Outlink;
-
-public class FetchNode {
- private Text url = null;
- private Outlink[] outlinks;
- private int status = 0;
- private String title = null;
- private long fetchTime = 0;
-
- public Text getUrl() {
- return url;
- }
- public void setUrl(Text url) {
- this.url = url;
- }
- public Outlink[] getOutlinks() {
- Outlink[] copyOutlinks = new Outlink[outlinks.length];
- for (int i = copyOutlinks.length-1; i>=0; i--) {
- Outlink o = outlinks[i];
- if (o != null) {
- try {
- copyOutlinks[i] = new Outlink(o.getToUrl(), o.getAnchor());
- } catch (MalformedURLException e) {
- // ignore
- }
- }
- }
- return outlinks;
- }
- public void setOutlinks(Outlink[] links) {
- this.outlinks = links;
- }
- public int getStatus() {
- return status;
- }
- public void setStatus(int status) {
- this.status = status;
- }
- public String getTitle() {
- return title;
- }
- public void setTitle(String title) {
- this.title = title;
- }
- public long getFetchTime() {
- return fetchTime;
- }
- public void setFetchTime(long fetchTime) {
- this.fetchTime = fetchTime;
- }
-}
diff --git a/src/java/org/apache/nutch/fetcher/FetchNodeDb.java b/src/java/org/apache/nutch/fetcher/FetchNodeDb.java
deleted file mode 100644
index 695dd90c7..000000000
--- a/src/java/org/apache/nutch/fetcher/FetchNodeDb.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.fetcher;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class FetchNodeDb {
-
- private Map<Integer, FetchNode> fetchNodeDbMap;
- private int index;
- private static FetchNodeDb fetchNodeDbInstance = null;
-
- public FetchNodeDb(){
- fetchNodeDbMap = new ConcurrentHashMap<>();
- index = 1;
- }
-
- public static FetchNodeDb getInstance(){
-
- if(fetchNodeDbInstance == null){
- fetchNodeDbInstance = new FetchNodeDb();
- }
- return fetchNodeDbInstance;
- }
-
- public void put(String url, FetchNode fetchNode){
- System.out.println("FetchNodeDb : putting node - " + fetchNode.hashCode());
- fetchNodeDbMap.put(index++, fetchNode);
- }
- public Map<Integer, FetchNode> getFetchNodeDb(){
- return fetchNodeDbMap;
- }
-}
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java
index d367a4f29..e3ee092b4 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -65,7 +65,6 @@ import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
-import org.apache.nutch.service.NutchServer;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.URLUtil;
import org.slf4j.Logger;
@@ -146,10 +145,6 @@ public class FetcherThread extends Thread {
private long robotsDeferVisitsDelay;
private int robotsDeferVisitsRetries;
- //Used by the REST service
- private FetchNode fetchNode;
- private boolean reportToNutchServer;
-
//Used for publishing events
private FetcherThreadPublisher publisher;
private boolean activatePublisher;
@@ -308,17 +303,7 @@ public class FetcherThread extends Thread {
FetchItem fit = null;
try {
- // checking for the server to be running and fetcher.parse to be true
- if (parsing && NutchServer.getInstance().isRunning())
- reportToNutchServer = true;
-
while (true) {
- // creating FetchNode for storing in FetchNodeDb
- if (reportToNutchServer)
- this.fetchNode = new FetchNode();
- else
- this.fetchNode = null;
-
// check whether must be stopped
if (isHalted()) {
LOG.debug("{} set to halted", getName());
@@ -443,13 +428,6 @@ public class FetcherThread extends Thread {
// unblock queue
fetchQueues.finishFetchItem(fit);
- // used for FetchNode
- if (fetchNode != null) {
- fetchNode.setStatus(status.getCode());
- fetchNode.setFetchTime(System.currentTimeMillis());
- fetchNode.setUrl(fit.url);
- }
-
//Publish fetch finish event
if(activatePublisher) {
FetcherThreadEvent endEvent = new FetcherThreadEvent(PublishEventType.END, fit.getUrl().toString());
@@ -816,13 +794,6 @@ public class FetcherThread extends Thread {
origin = originURL.getHost().toLowerCase();
}
}
-
- //used by fetchNode
- if(fetchNode!=null){
- fetchNode.setOutlinks(links);
- fetchNode.setTitle(parseData.getTitle());
- FetchNodeDb.getInstance().put(fetchNode.getUrl().toString(), fetchNode);
- }
int validCount = 0;
// Process all outlinks, normalize, filter and deduplicate
diff --git a/src/java/org/apache/nutch/indexer/IndexingJob.java b/src/java/org/apache/nutch/indexer/IndexingJob.java
index 224b4118e..8bf336880 100644
--- a/src/java/org/apache/nutch/indexer/IndexingJob.java
+++ b/src/java/org/apache/nutch/indexer/IndexingJob.java
@@ -309,7 +309,9 @@ public class IndexingJob extends NutchTool implements Tool {
}
- //Used for REST API
+ /**
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
+ */
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId) throws Exception {
boolean noCommit = false;
diff --git a/src/java/org/apache/nutch/metadata/Nutch.java b/src/java/org/apache/nutch/metadata/Nutch.java
index 0cfb26369..ce4cf519c 100644
--- a/src/java/org/apache/nutch/metadata/Nutch.java
+++ b/src/java/org/apache/nutch/metadata/Nutch.java
@@ -83,26 +83,23 @@ public interface Nutch {
public static final Text WRITABLE_FIXED_INTERVAL_KEY = new Text(
FIXED_INTERVAL_KEY);
- /** For progress of job. Used by the Nutch REST service */
+ /** For progress of job (programmatic / tooling). */
public static final String STAT_PROGRESS = "progress";
- /**Used by Nutch REST service */
+ /** Crawl id key for programmatic jobs. */
public static final String CRAWL_ID_KEY = "storage.crawl.id";
- /** Argument key to specify location of the seed url dir for the REST endpoints **/
+ /** Argument key for seed URL directory path. */
public static final String ARG_SEEDDIR = "url_dir";
- /** Argument key to specify name of a seed list for the REST endpoints **/
- public static final String ARG_SEEDNAME = "seedName";
- /** Argument key to specify the location of crawldb for the REST endpoints **/
+ /** Argument key for crawldb location in programmatic jobs. */
public static final String ARG_CRAWLDB = "crawldb";
- /** Argument key to specify the location of linkdb for the REST endpoints **/
+ /** Argument key for linkdb location in programmatic jobs. */
public static final String ARG_LINKDB = "linkdb";
- /** Name of the key used in the Result Map sent back by the REST endpoint **/
+ /** Name of the key used in the result map from {@link org.apache.nutch.util.NutchTool#run}. */
public static final String VAL_RESULT = "result";
- /** Argument key to specify the location of a directory of segments for the REST endpoints.
- * Similar to the -dir command in the bin/nutch script **/
+ /** Argument key for a directory of segments; similar to the -dir option in bin/nutch. */
public static final String ARG_SEGMENTDIR = "segment_dir";
- /** Argument key to specify the location of individual segment or list of segments for the REST endpoints. The behavior differs for diffirent endpoints: CrawlDb, LinkDb and Indexing Jobs take list of segments, Fetcher and Parse segment take one segment **/
+ /** Argument key for one segment or a list of segments (job-dependent). */
public static final String ARG_SEGMENTS = "segment";
- /** Argument key to specify the location of hostdb for the REST endpoints **/
+ /** Argument key for hostdb location in programmatic jobs. */
public static final String ARG_HOSTDB = "hostdb";
/** Title key in the Pub/Sub event metadata for the title of the parsed page*/
public static final String FETCH_EVENT_TITLE = "title";
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java
index 0b2a6f229..f4cb5e17b 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -325,8 +325,8 @@ public class ParseSegment extends NutchTool implements Tool {
return 0;
}
- /*
- * Used for Nutch REST service
+ /**
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
*/
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId) throws Exception {
diff --git a/src/java/org/apache/nutch/service/ConfManager.java b/src/java/org/apache/nutch/service/ConfManager.java
deleted file mode 100644
index fb4ec8758..000000000
--- a/src/java/org/apache/nutch/service/ConfManager.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.service.model.request.NutchConfig;
-
-public interface ConfManager {
-
- public Configuration get(String confId);
-
- public Map<String, String> getAsMap(String confId);
-
- public void setProperty(String confId, String propName, String propValue);
-
- public Set<String> list();
-
- public String create(NutchConfig nutchConfig);
-
- public void delete(String confId);
-}
diff --git a/src/java/org/apache/nutch/service/JobManager.java b/src/java/org/apache/nutch/service/JobManager.java
deleted file mode 100644
index 35a56e1e5..000000000
--- a/src/java/org/apache/nutch/service/JobManager.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.util.Collection;
-import org.apache.nutch.service.model.request.JobConfig;
-import org.apache.nutch.service.model.response.JobInfo;
-import org.apache.nutch.service.model.response.JobInfo.State;
-
-public interface JobManager {
-
- public static enum JobType{
- INJECT, GENERATE, FETCH, PARSE, UPDATEDB, INDEX, READDB, CLASS, INVERTLINKS, DEDUP
- };
- public Collection<JobInfo> list(String crawlId, State state);
-
- public JobInfo get(String crawlId, String id);
-
- /**
- * Creates specified job
- * @param jobConfig a job-specific {@link JobConfig}
- * @return JobInfo
- */
- public JobInfo create(JobConfig jobConfig);
-
- public boolean abort(String crawlId, String id);
-
- public boolean stop(String crawlId, String id);
-}
diff --git a/src/java/org/apache/nutch/service/NutchReader.java b/src/java/org/apache/nutch/service/NutchReader.java
deleted file mode 100644
index 8d7725424..000000000
--- a/src/java/org/apache/nutch/service/NutchReader.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.io.FileNotFoundException;
-import java.lang.invoke.MethodHandles;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.util.NutchConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public interface NutchReader {
-
- static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
- public static final Configuration conf = NutchConfiguration.create();
-
- public List<?> read(String path) throws FileNotFoundException;
- public List<?> head(String path, int nrows) throws FileNotFoundException;
- public List<?> slice(String path, int start, int end) throws FileNotFoundException;
- public int count(String path) throws FileNotFoundException;
-}
diff --git a/src/java/org/apache/nutch/service/NutchServer.java b/src/java/org/apache/nutch/service/NutchServer.java
deleted file mode 100644
index 946867031..000000000
--- a/src/java/org/apache/nutch/service/NutchServer.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider;
-
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.cxf.binding.BindingFactoryManager;
-import org.apache.cxf.jaxrs.JAXRSBindingFactory;
-import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
-import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
-import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
-import org.apache.nutch.fetcher.FetchNodeDb;
-import org.apache.nutch.service.impl.ConfManagerImpl;
-import org.apache.nutch.service.impl.JobFactory;
-import org.apache.nutch.service.impl.JobManagerImpl;
-import org.apache.nutch.service.impl.SeedManagerImpl;
-import org.apache.nutch.service.impl.NutchServerPoolExecutor;
-import org.apache.nutch.service.model.response.JobInfo;
-import org.apache.nutch.service.model.response.JobInfo.State;
-import org.apache.nutch.service.resources.AdminResource;
-import org.apache.nutch.service.resources.ConfigResource;
-import org.apache.nutch.service.resources.DbResource;
-import org.apache.nutch.service.resources.JobResource;
-import org.apache.nutch.service.resources.ReaderResouce;
-import org.apache.nutch.service.resources.SeedResource;
-import org.apache.nutch.service.resources.ServicesResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Queues;
-
-public class NutchServer {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String LOCALHOST = "localhost";
- private static final Integer DEFAULT_PORT = 8081;
- private static final int JOB_CAPACITY = 100;
-
- private static Integer port = DEFAULT_PORT;
- private static String host = LOCALHOST;
-
- private static final String CMD_HELP = "help";
- private static final String CMD_PORT = "port";
- private static final String CMD_HOST = "host";
-
- private long started;
- private boolean running;
- private ConfManager configManager;
- private JobManager jobManager;
- private SeedManager seedManager;
- private JAXRSServerFactoryBean sf;
-
- private static FetchNodeDb fetchNodeDb;
-
- private static NutchServer server;
-
- static {
- server = new NutchServer();
- }
-
- private NutchServer() {
- configManager = new ConfManagerImpl();
- seedManager = new SeedManagerImpl();
- BlockingQueue<Runnable> runnables = Queues.newArrayBlockingQueue(JOB_CAPACITY);
- NutchServerPoolExecutor executor = new NutchServerPoolExecutor(10, JOB_CAPACITY, 1, TimeUnit.HOURS, runnables);
- jobManager = new JobManagerImpl(new JobFactory(), configManager, executor);
- fetchNodeDb = FetchNodeDb.getInstance();
-
- sf = new JAXRSServerFactoryBean();
- BindingFactoryManager manager = sf.getBus().getExtension(BindingFactoryManager.class);
- JAXRSBindingFactory factory = new JAXRSBindingFactory();
- factory.setBus(sf.getBus());
- manager.registerBindingFactory(JAXRSBindingFactory.JAXRS_BINDING_ID, factory);
- sf.setResourceClasses(getClasses());
- sf.setResourceProviders(getResourceProviders());
- sf.setProvider(new JacksonJaxbJsonProvider());
-
- }
-
- public static NutchServer getInstance() {
- return server;
- }
-
- protected static void startServer() {
- server.start();
- }
-
- private void start() {
- LOG.info("Starting NutchServer on {}:{} ...", host, port);
- try{
- String address = "http://" + host + ":" + port;
- sf.setAddress(address);
- sf.create();
- }catch(Exception e){
- throw new IllegalStateException("Server could not be started", e);
- }
-
- started = System.currentTimeMillis();
- running = true;
- LOG.info("Started Nutch Server on {}:{} at {}", new Object[] {host, port, started});
- }
-
- private List<Class<?>> getClasses() {
- List<Class<?>> resources = new ArrayList<>();
- resources.add(JobResource.class);
- resources.add(ConfigResource.class);
- resources.add(DbResource.class);
- resources.add(AdminResource.class);
- resources.add(SeedResource.class);
- resources.add(ReaderResouce.class);
- resources.add(ServicesResource.class);
- return resources;
- }
-
- private List<ResourceProvider> getResourceProviders() {
- List<ResourceProvider> resourceProviders = new ArrayList<>();
- resourceProviders.add(new SingletonResourceProvider(getConfManager()));
- return resourceProviders;
- }
-
- public ConfManager getConfManager() {
- return configManager;
- }
-
- public JobManager getJobManager() {
- return jobManager;
- }
-
- public SeedManager getSeedManager() {
- return seedManager;
- }
-
- public FetchNodeDb getFetchNodeDb(){
- return fetchNodeDb;
- }
-
- public boolean isRunning(){
- return running;
- }
-
- public long getStarted(){
- return started;
- }
-
- public static void main(String[] args) throws ParseException {
- CommandLineParser parser = new PosixParser();
- Options options = createOptions();
- CommandLine commandLine = parser.parse(options, args);
- if (commandLine.hasOption(CMD_HELP)) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("NutchServer", options, true);
- return;
- }
-
- if (commandLine.hasOption(CMD_PORT)) {
- port = Integer.parseInt(commandLine.getOptionValue(CMD_PORT));
- }
-
- if (commandLine.hasOption(CMD_HOST)) {
- host = commandLine.getOptionValue(CMD_HOST);
- }
-
- startServer();
- }
-
- private static Options createOptions() {
- Options options = new Options();
-
- OptionBuilder.withDescription("Show this help");
- options.addOption(OptionBuilder.create(CMD_HELP));
-
- OptionBuilder.withArgName("port");
- OptionBuilder.hasOptionalArg();
- OptionBuilder.withDescription("The port to run the Nutch Server. Default port 8081");
- options.addOption(OptionBuilder.create(CMD_PORT));
-
- OptionBuilder.withArgName("host");
- OptionBuilder.hasOptionalArg();
- OptionBuilder.withDescription("The host to bind the Nutch Server to. Default is localhost.");
- options.addOption(OptionBuilder.create(CMD_HOST));
-
- return options;
- }
-
- public boolean canStop(boolean force){
- if(force)
- return true;
-
- Collection<JobInfo> jobs = getJobManager().list(null, State.RUNNING);
- return jobs.isEmpty();
- }
-
- protected static void setPort(int port) {
- NutchServer.port = port;
- }
-
- public int getPort() {
- return port;
- }
-
- public void stop() {
- System.exit(0);
- }
-}
diff --git a/src/java/org/apache/nutch/service/SeedManager.java b/src/java/org/apache/nutch/service/SeedManager.java
deleted file mode 100644
index 11ddedb62..000000000
--- a/src/java/org/apache/nutch/service/SeedManager.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.util.Map;
-
-import org.apache.nutch.service.model.request.SeedList;
-
-public interface SeedManager {
-
- public SeedList getSeedList(String seedName);
-
- public void setSeedList(String seedName, SeedList seedList);
-
- public boolean deleteSeedList(String seedName);
-
- public Map<String, SeedList> getSeeds();
-}
diff --git a/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java b/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java
deleted file mode 100644
index 7db24d319..000000000
--- a/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.service.ConfManager;
-import org.apache.nutch.service.model.request.NutchConfig;
-import org.apache.nutch.service.resources.ConfigResource;
-import org.apache.nutch.util.NutchConfiguration;
-
-import com.google.common.collect.Maps;
-
-public class ConfManagerImpl implements ConfManager {
-
-
- private Map<String, Configuration> configurations = Maps.newConcurrentMap();
-
- private AtomicInteger newConfigId = new AtomicInteger();
-
- public ConfManagerImpl() {
- configurations.put(ConfigResource.DEFAULT, NutchConfiguration.create());
- }
-
- /**
- * Returns the configuration associatedConfManagerImpl with the given confId
- */
- @Override
- public Configuration get(String confId) {
- if (confId == null) {
- return configurations.get(ConfigResource.DEFAULT);
- }
- return configurations.get(confId);
- }
-
- @Override
- public Map<String, String> getAsMap(String confId) {
- Configuration configuration = configurations.get(confId);
- if (configuration == null) {
- return Collections.emptyMap();
- }
-
- Iterator<Entry<String, String>> iterator = configuration.iterator();
- Map<String, String> configMap = Maps.newTreeMap();
- while (iterator.hasNext()) {
- Entry<String, String> entry = iterator.next();
- configMap.put(entry.getKey(), entry.getValue());
- }
- return configMap;
- }
-
- /**
- * Sets the given property in the configuration associated with the confId
- */
- @Override
- public void setProperty(String confId, String propName, String propValue) {
- if (!configurations.containsKey(confId)) {
- throw new IllegalArgumentException("Unknown configId '" + confId + "'");
- }
- Configuration conf = configurations.get(confId);
- conf.set(propName, propValue);
- }
-
- @Override
- public Set<String> list() {
- return configurations.keySet();
- }
-
- /**
- * Created a new configuration based on the values provided.
- * @param nutchConfig crawler configuration
- * @return String - confId
- */
- @Override
- public String create(NutchConfig nutchConfig) {
- if (StringUtils.isBlank(nutchConfig.getConfigId())) {
- nutchConfig.setConfigId(String.valueOf(newConfigId.incrementAndGet()));
- }
-
- if (!canCreate(nutchConfig)) {
- throw new IllegalArgumentException("Config already exists.");
- }
-
- createHadoopConfig(nutchConfig);
- return nutchConfig.getConfigId();
- }
-
-
- @Override
- public void delete(String confId) {
- configurations.remove(confId);
- }
-
- private boolean canCreate(NutchConfig nutchConfig) {
- if (nutchConfig.isForce()) {
- return true;
- }
- if (!configurations.containsKey(nutchConfig.getConfigId())) {
- return true;
- }
- return false;
- }
-
- private void createHadoopConfig(NutchConfig nutchConfig) {
- Configuration conf = NutchConfiguration.create();
- configurations.put(nutchConfig.getConfigId(), conf);
-
- if (MapUtils.isEmpty(nutchConfig.getParams())) {
- return;
- }
- for (Entry<String, String> e : nutchConfig.getParams().entrySet()) {
- conf.set(e.getKey(), e.getValue());
- }
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/impl/JobFactory.java b/src/java/org/apache/nutch/service/impl/JobFactory.java
deleted file mode 100644
index 60bbb253d..000000000
--- a/src/java/org/apache/nutch/service/impl/JobFactory.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.nutch.service.JobManager.JobType;
-import org.apache.nutch.crawl.CrawlDb;
-import org.apache.nutch.crawl.DeduplicationJob;
-import org.apache.nutch.crawl.Generator;
-import org.apache.nutch.crawl.Injector;
-import org.apache.nutch.crawl.LinkDb;
-import org.apache.nutch.fetcher.Fetcher;
-import org.apache.nutch.indexer.IndexingJob;
-import org.apache.nutch.parse.ParseSegment;
-import org.apache.nutch.util.NutchTool;
-
-import com.google.common.collect.Maps;
-
-public class JobFactory {
- private static Map<JobType, Class<? extends NutchTool>> typeToClass;
-
- static {
- typeToClass = Maps.newHashMap();
- typeToClass.put(JobType.INJECT, Injector.class);
- typeToClass.put(JobType.GENERATE, Generator.class);
- typeToClass.put(JobType.FETCH, Fetcher.class);
- typeToClass.put(JobType.PARSE, ParseSegment.class);
- typeToClass.put(JobType.INDEX, IndexingJob.class);
- typeToClass.put(JobType.UPDATEDB, CrawlDb.class);
- typeToClass.put(JobType.INVERTLINKS, LinkDb.class);
- typeToClass.put(JobType.DEDUP, DeduplicationJob.class);
- }
-
- public NutchTool createToolByType(JobType type, Configuration conf) {
- if (!typeToClass.containsKey(type)) {
- return null;
- }
- Class<? extends NutchTool> clz = typeToClass.get(type);
- return createTool(clz, conf);
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public NutchTool createToolByClassName(String className, Configuration conf) {
- try {
- Class clz = Class.forName(className);
- return createTool(clz, conf);
- } catch (ClassNotFoundException e) {
- throw new IllegalStateException(e);
- }
- }
-
- private NutchTool createTool(Class<? extends NutchTool> clz,
- Configuration conf) {
- return ReflectionUtils.newInstance(clz, conf);
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/impl/JobManagerImpl.java b/src/java/org/apache/nutch/service/impl/JobManagerImpl.java
deleted file mode 100644
index 3bcb7dde9..000000000
--- a/src/java/org/apache/nutch/service/impl/JobManagerImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.util.Collection;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.service.ConfManager;
-import org.apache.nutch.service.JobManager;
-import org.apache.nutch.service.model.request.JobConfig;
-import org.apache.nutch.service.model.response.JobInfo;
-import org.apache.nutch.service.model.response.JobInfo.State;
-import org.apache.nutch.util.NutchTool;
-
-public class JobManagerImpl implements JobManager {
-
- private JobFactory jobFactory;
- private NutchServerPoolExecutor executor;
- private ConfManager configManager;
-
- public JobManagerImpl(JobFactory jobFactory, ConfManager configManager, NutchServerPoolExecutor executor) {
- this.jobFactory = jobFactory;
- this.configManager = configManager;
- this.executor = executor;
- }
-
- @Override
- public JobInfo create(JobConfig jobConfig) {
- if (jobConfig.getArgs() == null) {
- throw new IllegalArgumentException("Arguments cannot be null!");
- }
- Configuration conf = cloneConfiguration(jobConfig.getConfId());
- NutchTool tool = createTool(jobConfig, conf);
- JobWorker worker = new JobWorker(jobConfig, conf, tool);
- executor.execute(worker);
- executor.purge();
- return worker.getInfo();
- }
-
- private Configuration cloneConfiguration(String confId) {
- Configuration conf = configManager.get(confId);
- if (conf == null) {
- throw new IllegalArgumentException("Unknown confId " + confId);
- }
- return new Configuration(conf);
- }
-
- @Override
- public Collection<JobInfo> list(String crawlId, State state) {
- if (state == null || state == State.ANY) {
- return executor.getAllJobs();
- }
- if (state == State.RUNNING || state == State.IDLE) {
- return executor.getJobRunning();
- }
- return executor.getJobHistory();
- }
-
- @Override
- public JobInfo get(String crawlId, String jobId) {
- return executor.getInfo(jobId);
- }
-
- @Override
- public boolean abort(String crawlId, String id) {
- return executor.findWorker(id).killJob();
- }
-
- @Override
- public boolean stop(String crawlId, String id) {
- return executor.findWorker(id).stopJob();
- }
-
- private NutchTool createTool(JobConfig jobConfig, Configuration conf){
- if(StringUtils.isNotBlank(jobConfig.getJobClassName())){
- return jobFactory.createToolByClassName(jobConfig.getJobClassName(), conf);
- }
- return jobFactory.createToolByType(jobConfig.getType(), conf);
- }
-}
diff --git a/src/java/org/apache/nutch/service/impl/JobWorker.java b/src/java/org/apache/nutch/service/impl/JobWorker.java
deleted file mode 100644
index d3343ae71..000000000
--- a/src/java/org/apache/nutch/service/impl/JobWorker.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.lang.invoke.MethodHandles;
-import java.text.MessageFormat;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.metadata.Nutch;
-import org.apache.nutch.service.model.request.JobConfig;
-import org.apache.nutch.service.model.response.JobInfo;
-import org.apache.nutch.service.model.response.JobInfo.State;
-import org.apache.nutch.service.resources.ConfigResource;
-import org.apache.nutch.util.NutchTool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class JobWorker implements Runnable{
-
- private JobInfo jobInfo;
- private JobConfig jobConfig;
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
- private NutchTool tool;
-
- /**
- * To initialize JobWorker thread with the Job Configurations provided by user.
- * @param jobConfig job-specific {@link JobConfig}
- * @param conf a populated {@link Configuration}
- * @param tool {!{@link NutchTool} to run
- * return JobWorker
- */
- public JobWorker(JobConfig jobConfig, Configuration conf, NutchTool tool) {
- this.jobConfig = jobConfig;
- this.tool = tool;
- if (jobConfig.getConfId() == null) {
- jobConfig.setConfId(ConfigResource.DEFAULT);
- }
-
- jobInfo = new JobInfo(generateId(), jobConfig, State.IDLE, "idle");
- if (jobConfig.getCrawlId() != null) {
- conf.set(Nutch.CRAWL_ID_KEY, jobConfig.getCrawlId());
- }
- }
-
- private String generateId() {
- if (jobConfig.getCrawlId() == null) {
- return MessageFormat.format("{0}-{1}-{2}", jobConfig.getConfId(),
- jobConfig.getType(), String.valueOf(hashCode()));
- }
- return MessageFormat.format("{0}-{1}-{2}-{3}", jobConfig.getCrawlId(),
- jobConfig.getConfId(), jobConfig.getType(), String.valueOf(hashCode()));
- }
-
- @Override
- public void run() {
- try {
- getInfo().setState(State.RUNNING);
- getInfo().setMsg("OK");
- getInfo().setResult(tool.run(getInfo().getArgs(), getInfo().getCrawlId()));
- getInfo().setState(State.FINISHED);
- } catch (Exception e) {
- LOG.error("Cannot run job worker!", e);
- getInfo().setMsg("ERROR: " + e.toString());
- getInfo().setState(State.FAILED);
- }
- }
-
- public JobInfo getInfo() {
- return jobInfo;
- }
-
- /**
- * To stop the executing job
- * @return boolean true/false
- */
- public boolean stopJob() {
- getInfo().setState(State.STOPPING);
- try {
- return tool.stopJob();
- } catch (Exception e) {
- throw new RuntimeException(
- "Cannot stop job with id " + getInfo().getId(), e);
- }
- }
-
- public boolean killJob() {
- getInfo().setState(State.KILLING);
- try {
- boolean result = tool.killJob();
- getInfo().setState(State.KILLED);
- return result;
- } catch (Exception e) {
- throw new RuntimeException(
- "Cannot kill job with id " + getInfo().getId(), e);
- }
- }
-
- public void setInfo(JobInfo jobInfo) {
- this.jobInfo = jobInfo;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/impl/LinkReader.java b/src/java/org/apache/nutch/service/impl/LinkReader.java
deleted file mode 100644
index 59d84509a..000000000
--- a/src/java/org/apache/nutch/service/impl/LinkReader.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import javax.ws.rs.WebApplicationException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.nutch.scoring.webgraph.LinkDatum;
-import org.apache.nutch.service.NutchReader;
-
-public class LinkReader implements NutchReader {
-
- @Override
- public List<HashMap<String, String>> read(String path) throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- LinkDatum value = new LinkDatum();
-
- while(reader.next(key, value)) {
- try {
- HashMap<String, String> t_row = getLinksRow(key,value);
- rows.add(t_row);
- }
- catch (Exception e) {
- }
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
- }
-
- @Override
- public List<HashMap<String, String>> head(String path, int nrows) throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- LinkDatum value = new LinkDatum();
- int i = 0;
- while(reader.next(key, value) && i<nrows) {
-
- HashMap<String, String> t_row = getLinksRow(key,value);
- rows.add(t_row);
-
- i++;
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
- }
-
- @Override
- public List<HashMap<String, String>> slice(String path, int start, int end)
- throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- LinkDatum value = new LinkDatum();
- int i = 0;
-
- for(;i<start && reader.next(key, value);i++){} // increment to read start position
- while(reader.next(key, value) && i<end) {
- HashMap<String, String> t_row = getLinksRow(key,value);
- rows.add(t_row);
-
- i++;
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
- }
-
- @Override
- public int count(String path) throws FileNotFoundException {
- Path file = new Path(path);
- SequenceFile.Reader reader;
- int i = 0;
- try {
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
-
- while(reader.next(key, value)) {
- i++;
- }
- reader.close();
- } catch(FileNotFoundException fne){
- throw new FileNotFoundException();
- }catch (IOException e) {
- LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
- return i;
- }
-
- private HashMap<String, String> getLinksRow(Writable key, LinkDatum value) {
- HashMap<String, String> tRow = new HashMap<>();
- tRow.put("key_url", key.toString());
- tRow.put("url", value.getUrl());
- tRow.put("anchor", value.getAnchor());
- tRow.put("score", String.valueOf(value.getScore()));
- tRow.put("timestamp", String.valueOf(value.getTimestamp()));
- tRow.put("linktype", String.valueOf(value.getLinkType()));
-
- return tRow;
- }
-}
diff --git a/src/java/org/apache/nutch/service/impl/NodeReader.java b/src/java/org/apache/nutch/service/impl/NodeReader.java
deleted file mode 100644
index efa94f232..000000000
--- a/src/java/org/apache/nutch/service/impl/NodeReader.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import javax.ws.rs.WebApplicationException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.nutch.scoring.webgraph.Node;
-import org.apache.nutch.service.NutchReader;
-
-public class NodeReader implements NutchReader {
-
- @Override
- public List<HashMap<String, String>> read(String path) throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Node value = new Node();
-
- while(reader.next(key, value)) {
- try {
- HashMap<String, String> t_row = getNodeRow(key,value);
- rows.add(t_row);
- }
- catch (Exception e) {
- }
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
-
- }
-
- @Override
- public List<HashMap<String, String>> head(String path, int nrows) throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Node value = new Node();
- int i = 0;
- while(reader.next(key, value) && i<nrows) {
- HashMap<String, String> t_row = getNodeRow(key,value);
- rows.add(t_row);
-
- i++;
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
- }
-
- @Override
- public List<HashMap<String, String>> slice(String path, int start, int end)
- throws FileNotFoundException {
- List<HashMap<String, String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Node value = new Node();
- int i = 0;
-
- for(;i<start && reader.next(key, value);i++){} // increment to read start position
- while(reader.next(key, value) && i<end) {
- HashMap<String, String> t_row = getNodeRow(key,value);
- rows.add(t_row);
-
- i++;
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return rows;
- }
-
- @Override
- public int count(String path) throws FileNotFoundException {
- Path file = new Path(path);
- SequenceFile.Reader reader;
- int i =0;
- try{
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key = (Writable)
- ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Node value = new Node();
-
- while(reader.next(key, value)) {
- i++;
- }
- reader.close();
-
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
-
- }catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
-
- return i;
- }
-
- private HashMap<String, String> getNodeRow(Writable key, Node value) {
- HashMap<String, String> tRow = new HashMap<>();
- tRow.put("key_url", key.toString());
- tRow.put("num_inlinks", String.valueOf(value.getNumInlinks()) );
- tRow.put("num_outlinks", String.valueOf(value.getNumOutlinks()) );
- tRow.put("inlink_score", String.valueOf(value.getInlinkScore()));
- tRow.put("outlink_score", String.valueOf(value.getOutlinkScore()));
- tRow.put("metadata", value.getMetadata().toString());
-
- return tRow;
- }
-}
diff --git a/src/java/org/apache/nutch/service/impl/NutchServerPoolExecutor.java b/src/java/org/apache/nutch/service/impl/NutchServerPoolExecutor.java
deleted file mode 100644
index 1d1e8175b..000000000
--- a/src/java/org/apache/nutch/service/impl/NutchServerPoolExecutor.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nutch.service.model.response.JobInfo;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Queues;
-
-public class NutchServerPoolExecutor extends ThreadPoolExecutor{
-
- private Queue<JobWorker> workersHistory;
- private Queue<JobWorker> runningWorkers;
-
- public NutchServerPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue){
- super(corePoolSize, maxPoolSize, keepAliveTime, unit, workQueue);
- workersHistory = Queues.newArrayBlockingQueue(maxPoolSize);
- runningWorkers = Queues.newArrayBlockingQueue(maxPoolSize);
- }
-
- @Override
- protected void beforeExecute(Thread thread, Runnable runnable) {
- super.beforeExecute(thread, runnable);
- synchronized (runningWorkers) {
- runningWorkers.offer(((JobWorker) runnable));
- }
- }
-
- @Override
- protected void afterExecute(Runnable runnable, Throwable throwable) {
- super.afterExecute(runnable, throwable);
- synchronized (runningWorkers) {
- runningWorkers.remove((JobWorker) runnable);
- }
- JobWorker worker = ((JobWorker) runnable);
- addStatusToHistory(worker);
- }
-
- private void addStatusToHistory(JobWorker worker) {
- synchronized (workersHistory) {
- if (!workersHistory.offer(worker)) {
- workersHistory.poll();
- workersHistory.add(worker);
- }
- }
- }
-
- /**
- * Find the Job Worker Thread.
- * @param jobId a jobId allows locating a specific worker thread
- * @return a {@link JobWorker} or else null
- */
- public JobWorker findWorker(String jobId) {
- synchronized (runningWorkers) {
- for (JobWorker worker : runningWorkers) {
- if (StringUtils.equals(worker.getInfo().getId(), jobId)) {
- return worker;
- }
- }
- }
- return null;
- }
-
- /**
- * Get the Job history
- * @return a {@link Collection} of {@link JobInfo}'s
- */
- public Collection<JobInfo> getJobHistory() {
- return getJobsInfo(workersHistory);
- }
-
- /**
- * Get the list of currently running jobs
- * @return a {@link Collection} of {@link JobInfo}'s
- */
- public Collection<JobInfo> getJobRunning() {
- return getJobsInfo(runningWorkers);
- }
-
- /**
- * get all jobs (currently running and completed)
- * @return a {@link Collection} of {@link JobInfo}'s
- */
- @SuppressWarnings("unchecked")
- public Collection<JobInfo> getAllJobs() {
- return CollectionUtils.union(getJobRunning(), getJobHistory());
- }
-
- private Collection<JobInfo> getJobsInfo(Collection<JobWorker> workers) {
- List<JobInfo> jobsInfo = Lists.newLinkedList();
- for (JobWorker worker : workers) {
- jobsInfo.add(worker.getInfo());
- }
- return jobsInfo;
- }
-
-
- public JobInfo getInfo(String jobId) {
- for (JobInfo jobInfo : getAllJobs()) {
- if (StringUtils.equals(jobId, jobInfo.getId())) {
- return jobInfo;
- }
- }
- return null;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/impl/SeedManagerImpl.java b/src/java/org/apache/nutch/service/impl/SeedManagerImpl.java
deleted file mode 100644
index a28de943c..000000000
--- a/src/java/org/apache/nutch/service/impl/SeedManagerImpl.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.nutch.service.SeedManager;
-import org.apache.nutch.service.model.request.SeedList;
-
-public class SeedManagerImpl implements SeedManager {
-
- private static Map<String, SeedList> seeds;
-
- public SeedManagerImpl() {
- seeds = new HashMap<>();
- }
-
- @Override
- public SeedList getSeedList(String seedName) {
- if(seeds.containsKey(seedName)) {
- return seeds.get(seedName);
- }
- else
- return null;
- }
-
- @Override
- public void setSeedList(String seedName, SeedList seedList) {
- seeds.put(seedName, seedList);
- }
-
- @Override
- public Map<String, SeedList> getSeeds(){
- return seeds;
- }
-
- @Override
- public boolean deleteSeedList(String seedName) {
- if(seeds.containsKey(seedName)) {
- seeds.remove(seedName);
- return true;
- }
- else
- return false;
- }
-}
diff --git a/src/java/org/apache/nutch/service/impl/SequenceReader.java b/src/java/org/apache/nutch/service/impl/SequenceReader.java
deleted file mode 100644
index 26b3d55d4..000000000
--- a/src/java/org/apache/nutch/service/impl/SequenceReader.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.ws.rs.WebApplicationException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.nutch.service.NutchReader;
-
-/**
- * Enables reading a sequence file and methods provide different
- * ways to read the file.
- * @author Sujen Shah
- *
- */
-public class SequenceReader implements NutchReader {
-
- @Override
- public List<List<String>> read(String path) throws FileNotFoundException {
- // TODO Auto-generated method stub
- List<List<String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try {
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key =
- (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value =
- (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
-
- while(reader.next(key, value)) {
- List<String> row = new ArrayList<>();
- row.add(key.toString());
- row.add(value.toString());
- rows.add(row);
- }
- reader.close();
- }catch(FileNotFoundException fne){
- throw new FileNotFoundException();
- }catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
- return rows;
- }
-
- @Override
- public List<List<String>> head(String path, int nrows)
- throws FileNotFoundException {
- // TODO Auto-generated method stub
-
- List<List<String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try {
-
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key =
- (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value =
- (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
- int i = 0;
- while(reader.next(key, value) && i<nrows) {
- List<String> row = new ArrayList<>();
- row.add(key.toString());
- row.add(value.toString());
- rows.add(row);
- i++;
- }
- reader.close();
- } catch(FileNotFoundException fne){
- throw new FileNotFoundException();
- }catch (IOException e) {
- // TODO Auto-generated catch block
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
- return rows;
- }
-
- @Override
- public List<List<String>> slice(String path, int start, int end)
- throws FileNotFoundException {
- List<List<String>> rows= new ArrayList<>();
- Path file = new Path(path);
- SequenceFile.Reader reader;
- try {
-
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key =
- (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value =
- (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
- int i = 0;
-
- for(;i<start && reader.next(key, value);i++){} // increment to read start position
- while(reader.next(key, value) && i<end) {
- List<String> row = new ArrayList<>();
- row.add(key.toString());
- row.add(value.toString());
- rows.add(row);
- i++;
- }
- reader.close();
- } catch(FileNotFoundException fne){
- throw new FileNotFoundException();
- }catch (IOException e) {
- // TODO Auto-generated catch block
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
- return rows;
- }
-
- @Override
- public int count(String path) throws FileNotFoundException {
- Path file = new Path(path);
- SequenceFile.Reader reader;
- int i = 0;
- try {
- reader = new SequenceFile.Reader(conf, Reader.file(file));
- Writable key =
- (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value =
- (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
-
- while(reader.next(key, value)) {
- i++;
- }
- reader.close();
- } catch(FileNotFoundException fne){
- throw new FileNotFoundException();
- }catch (IOException e) {
- // TODO Auto-generated catch block
- LOG.error("Error occurred while reading file {} : {}", file,
- StringUtils.stringifyException(e));
- throw new WebApplicationException();
- }
- return i;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/impl/ServiceWorker.java b/src/java/org/apache/nutch/service/impl/ServiceWorker.java
deleted file mode 100644
index f86acadde..000000000
--- a/src/java/org/apache/nutch/service/impl/ServiceWorker.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.impl;
-
-import java.lang.invoke.MethodHandles;
-
-import org.apache.hadoop.util.StringUtils;
-import org.apache.nutch.service.model.request.ServiceConfig;
-import org.apache.nutch.util.NutchTool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ServiceWorker implements Runnable {
-
- private ServiceConfig serviceConfig;
- private NutchTool tool;
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
-
- public ServiceWorker(ServiceConfig serviceConfig, NutchTool tool) {
- this.serviceConfig = serviceConfig;
- this.tool = tool;
- }
-
- @Override
- public void run() {
- try {
- tool.run(serviceConfig.getArgs(), serviceConfig.getCrawlId());
- } catch (Exception e) {
- // TODO Auto-generated catch block
- LOG.error("Error running service worker : {}",
- StringUtils.stringifyException(e));
- }
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/model/request/DbQuery.java b/src/java/org/apache/nutch/service/model/request/DbQuery.java
deleted file mode 100644
index 4b707df20..000000000
--- a/src/java/org/apache/nutch/service/model/request/DbQuery.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class DbQuery {
-
- private String confId;
- private String type;
- private Map<String, String> args = new HashMap<>();
- private String crawlId;
-
- public String getConfId() {
- return confId;
- }
- public void setConfId(String confId) {
- this.confId = confId;
- }
- public Map<String, String> getArgs() {
- return args;
- }
- public void setArgs(Map<String, String> args) {
- this.args = args;
- }
- public String getType() {
- return type;
- }
- public void setType(String type) {
- this.type = type;
- }
- public String getCrawlId() {
- return crawlId;
- }
- public void setCrawlId(String crawlId) {
- this.crawlId = crawlId;
- }
-
-
-
-}
diff --git a/src/java/org/apache/nutch/service/model/request/JobConfig.java b/src/java/org/apache/nutch/service/model/request/JobConfig.java
deleted file mode 100644
index ab8051752..000000000
--- a/src/java/org/apache/nutch/service/model/request/JobConfig.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.util.Map;
-
-import org.apache.nutch.service.JobManager.JobType;
-
-/**
- * Job-specific configuration.
- *
- */
-public class JobConfig {
- private String crawlId;
- private JobType type;
- private String confId;
- private String jobClassName;
- private Map<String, Object> args;
-
- public String getCrawlId() {
- return crawlId;
- }
-
- public void setCrawlId(String crawlId) {
- this.crawlId = crawlId;
- }
-
- public JobType getType() {
- return type;
- }
-
- public void setType(JobType type) {
- this.type = type;
- }
-
- public String getConfId() {
- return confId;
- }
-
- public void setConfId(String confId) {
- this.confId = confId;
- }
-
- public Map<String, Object> getArgs() {
- return args;
- }
-
- public void setArgs(Map<String, Object> args) {
- this.args = args;
- }
-
- public String getJobClassName() {
- return jobClassName;
- }
-
- public void setJobClassName(String jobClass) {
- this.jobClassName = jobClass;
- }
-}
diff --git a/src/java/org/apache/nutch/service/model/request/NutchConfig.java b/src/java/org/apache/nutch/service/model/request/NutchConfig.java
deleted file mode 100644
index 7049463a8..000000000
--- a/src/java/org/apache/nutch/service/model/request/NutchConfig.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.util.Map;
-
-import java.util.Collections;
-
-public class NutchConfig {
- private String configId;
- private boolean force = false;
- private Map<String, String> params = Collections.emptyMap();
-
- public Map<String, String> getParams() {
- return params;
- }
-
- public void setParams(Map<String, String> params) {
- this.params = params;
- }
-
- public String getConfigId() {
- return configId;
- }
-
- public void setConfigId(String configId) {
- this.configId = configId;
- }
-
- public boolean isForce() {
- return force;
- }
-
- public void setForce(boolean force) {
- this.force = force;
- }
-}
diff --git a/src/java/org/apache/nutch/service/model/request/ReaderConfig.java b/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
deleted file mode 100644
index 3e44f879e..000000000
--- a/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-public class ReaderConfig {
-
- private String path;
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-}
diff --git a/src/java/org/apache/nutch/service/model/request/SeedList.java b/src/java/org/apache/nutch/service/model/request/SeedList.java
deleted file mode 100644
index 5bd3c4fa6..000000000
--- a/src/java/org/apache/nutch/service/model/request/SeedList.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.io.Serializable;
-import java.util.Collection;
-
-import org.apache.commons.collections4.CollectionUtils;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-
-public class SeedList implements Serializable {
-
- private Long id;
-
- private String name;
- private String seedFilePath;
-
-
- @JsonManagedReference
- private Collection<SeedUrl> seedUrls;
-
- public Long getId() {
- return id;
- }
-
- public void setId(Long id) {
- this.id = id;
- }
-
- public Collection<SeedUrl> getSeedUrls() {
- return seedUrls;
- }
-
- public void setSeedUrls(Collection<SeedUrl> seedUrls) {
- this.seedUrls = seedUrls;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getSeedFilePath() {
- return seedFilePath;
- }
-
- public void setSeedFilePath(String seedFilePath) {
- this.seedFilePath = seedFilePath;
- }
-
- @JsonIgnore
- public int getSeedUrlsCount() {
- if (CollectionUtils.isEmpty(seedUrls)) {
- return 0;
- }
- return seedUrls.size();
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((id == null) ? 0 : id.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- SeedList other = (SeedList) obj;
- if (id == null) {
- if (other.id != null)
- return false;
- } else if (!id.equals(other.id))
- return false;
- return true;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/model/request/SeedUrl.java b/src/java/org/apache/nutch/service/model/request/SeedUrl.java
deleted file mode 100644
index f05e4d025..000000000
--- a/src/java/org/apache/nutch/service/model/request/SeedUrl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-public class SeedUrl implements Serializable {
-
- private Long id;
-
- @JsonBackReference
- private SeedList seedList;
-
- private String url;
-
- public SeedUrl() {}
-
- public SeedUrl(String url) {
- this.url = url;
- }
-
- public Long getId() {
- return id;
- }
-
- public void setId(Long id) {
- this.id = id;
- }
-
- public String getUrl() {
- return url;
- }
-
- public void setUrl(String url) {
- this.url = url;
- }
-
- @JsonIgnore
- public SeedList getSeedList() {
- return seedList;
- }
-
- @JsonIgnore
- public void setSeedList(SeedList seedList) {
- this.seedList = seedList;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((id == null) ? 0 : id.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- SeedUrl other = (SeedUrl) obj;
- if (id == null) {
- if (other.id != null)
- return false;
- } else if (!id.equals(other.id))
- return false;
- return true;
- }
-}
diff --git a/src/java/org/apache/nutch/service/model/request/ServiceConfig.java b/src/java/org/apache/nutch/service/model/request/ServiceConfig.java
deleted file mode 100644
index 85d6a3ea6..000000000
--- a/src/java/org/apache/nutch/service/model/request/ServiceConfig.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.request;
-
-import java.util.Map;
-
-public class ServiceConfig {
-
- private String crawlId;
- private String confId;
- private Map<String, Object> args;
-
- public String getCrawlId() {
- return crawlId;
- }
-
- public void setCrawlId(String crawlId) {
- this.crawlId = crawlId;
- }
-
- public String getConfId() {
- return confId;
- }
-
- public void setConfId(String confId) {
- this.confId = confId;
- }
-
- public Map<String, Object> getArgs() {
- return args;
- }
-
- public void setArgs(Map<String, Object> args) {
- this.args = args;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/model/response/FetchNodeDbInfo.java b/src/java/org/apache/nutch/service/model/response/FetchNodeDbInfo.java
deleted file mode 100644
index 21887ad65..000000000
--- a/src/java/org/apache/nutch/service/model/response/FetchNodeDbInfo.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.response;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.nutch.parse.Outlink;
-
-public class FetchNodeDbInfo {
-
- private String url;
- private int status;
- private int numOfOutlinks;
- private List<ChildNode> children = new ArrayList<>();
-
-
- public String getUrl() {
- return url;
- }
-
-
- public void setUrl(String url) {
- this.url = url;
- }
-
-
- public int getStatus() {
- return status;
- }
-
-
- public void setStatus(int status) {
- this.status = status;
- }
-
-
- public int getNumOfOutlinks() {
- return numOfOutlinks;
- }
-
-
- public void setNumOfOutlinks(int numOfOutlinks) {
- this.numOfOutlinks = numOfOutlinks;
- }
-
- public void setChildNodes(Outlink[] links){
- ChildNode childNode;
- for(Outlink outlink: links){
- childNode = new ChildNode(outlink.getToUrl(), outlink.getAnchor());
- children.add(childNode);
- }
- }
-
-
- private class ChildNode{
- private String childUrl;
- private String anchorText;
-
- public ChildNode(String childUrl, String anchorText){
- this.childUrl = childUrl;
- this.anchorText = anchorText;
- }
-
- @SuppressWarnings("unused")
- public String getAnchorText() {
- return anchorText;
- }
- @SuppressWarnings("unused")
- public void setAnchorText(String anchorText) {
- this.anchorText = anchorText;
- }
- @SuppressWarnings("unused")
- public String getChildUrl() {
- return childUrl;
- }
- @SuppressWarnings("unused")
- public void setChildUrl(String childUrl) {
- this.childUrl = childUrl;
- }
- }
-
-
- public List<ChildNode> getChildren() {
- return children;
- }
-
-
- public void setChildren(List<ChildNode> children) {
- this.children = children;
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/model/response/JobInfo.java b/src/java/org/apache/nutch/service/model/response/JobInfo.java
deleted file mode 100644
index 807c2d5df..000000000
--- a/src/java/org/apache/nutch/service/model/response/JobInfo.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.response;
-
-import java.util.Map;
-
-import org.apache.nutch.service.JobManager.JobType;
-import org.apache.nutch.service.model.request.JobConfig;
-
-/**
- * This is the response object containing Job information
- */
-public class JobInfo {
-
- public static enum State {
- IDLE, RUNNING, FINISHED, FAILED, KILLED, STOPPING, KILLING, ANY
- };
-
- private String id;
- private JobType type;
- private String confId;
- private Map<String, Object> args;
- private Map<String, Object> result;
- private State state;
- private String msg;
- private String crawlId;
-
- public JobInfo(String generateId, JobConfig jobConfig, State state,
- String msg) {
- this.id = generateId;
- this.type = jobConfig.getType();
- this.confId = jobConfig.getConfId();
- this.crawlId = jobConfig.getCrawlId();
- this.args = jobConfig.getArgs();
- this.msg = msg;
- this.state = state;
- }
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
- public JobType getType() {
- return type;
- }
- public void setType(JobType type) {
- this.type = type;
- }
- public String getConfId() {
- return confId;
- }
- public void setConfId(String confId) {
- this.confId = confId;
- }
- public Map<String, Object> getArgs() {
- return args;
- }
- public void setArgs(Map<String, Object> args) {
- this.args = args;
- }
- public Map<String, Object> getResult() {
- return result;
- }
- public void setResult(Map<String, Object> result) {
- this.result = result;
- }
- public State getState() {
- return state;
- }
- public void setState(State state) {
- this.state = state;
- }
- public String getMsg() {
- return msg;
- }
- public void setMsg(String msg) {
- this.msg = msg;
- }
- public String getCrawlId() {
- return crawlId;
- }
- public void setCrawlId(String crawlId) {
- this.crawlId = crawlId;
- }
-}
diff --git a/src/java/org/apache/nutch/service/model/response/NutchServerInfo.java b/src/java/org/apache/nutch/service/model/response/NutchServerInfo.java
deleted file mode 100644
index f8867e6ce..000000000
--- a/src/java/org/apache/nutch/service/model/response/NutchServerInfo.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.response;
-
-import java.util.Collection;
-import java.util.Date;
-import java.util.Set;
-
-public class NutchServerInfo {
-
- private Date startDate;
- private Set<String> configuration;
- private Collection<JobInfo> jobs;
- private Collection<JobInfo> runningJobs;
- public Date getStartDate() {
- return startDate;
- }
- public void setStartDate(Date startDate) {
- this.startDate = startDate;
- }
- public Set<String> getConfiguration() {
- return configuration;
- }
- public void setConfiguration(Set<String> configuration) {
- this.configuration = configuration;
- }
- public Collection<JobInfo> getJobs() {
- return jobs;
- }
- public void setJobs(Collection<JobInfo> jobs) {
- this.jobs = jobs;
- }
- public Collection<JobInfo> getRunningJobs() {
- return runningJobs;
- }
- public void setRunningJobs(Collection<JobInfo> runningJobs) {
- this.runningJobs = runningJobs;
- }
-
-
-}
diff --git a/src/java/org/apache/nutch/service/model/response/ServiceInfo.java b/src/java/org/apache/nutch/service/model/response/ServiceInfo.java
deleted file mode 100644
index 456f8c53a..000000000
--- a/src/java/org/apache/nutch/service/model/response/ServiceInfo.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.model.response;
-
-import java.util.List;
-
-public class ServiceInfo {
-
- private List<String> dumpPaths;
-
- public List<String> getDumpPaths() {
- return dumpPaths;
- }
-
- public void setDumpPaths(List<String> dumpPaths) {
- this.dumpPaths = dumpPaths;
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/AbstractResource.java b/src/java/org/apache/nutch/service/resources/AbstractResource.java
deleted file mode 100644
index b277a7539..000000000
--- a/src/java/org/apache/nutch/service/resources/AbstractResource.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.nutch.service.ConfManager;
-import org.apache.nutch.service.JobManager;
-import org.apache.nutch.service.NutchServer;
-
-@Produces(MediaType.APPLICATION_JSON)
-public abstract class AbstractResource {
-
- protected JobManager jobManager;
- protected ConfManager configManager;
- protected NutchServer server;
-
- public AbstractResource() {
- server = NutchServer.getInstance();
- configManager = NutchServer.getInstance().getConfManager();
- jobManager = NutchServer.getInstance().getJobManager();
- }
-
- protected void throwBadRequestException(String message) {
- throw new WebApplicationException(Response.status(Status.BAD_REQUEST).entity(message).build());
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/AdminResource.java b/src/java/org/apache/nutch/service/resources/AdminResource.java
deleted file mode 100644
index 03832628a..000000000
--- a/src/java/org/apache/nutch/service/resources/AdminResource.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Date;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.QueryParam;
-
-import org.apache.nutch.service.model.response.JobInfo.State;
-import org.apache.nutch.service.model.response.NutchServerInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Path(value="/admin")
-public class AdminResource extends AbstractResource{
-
- private final int DELAY_SEC = 1;
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
-
- /**
- * Get the status of the Nutch Server
- * @return {@link NutchServerInfo} for the running service
- */
- @GET
- @Path(value="/")
- public NutchServerInfo getServerStatus(){
- NutchServerInfo serverInfo = new NutchServerInfo();
- serverInfo.setConfiguration(configManager.list());
- serverInfo.setStartDate(new Date(server.getStarted()));
- serverInfo.setJobs(jobManager.list(null, State.ANY));
- serverInfo.setRunningJobs(jobManager.list(null, State.RUNNING));
- return serverInfo;
- }
-
- /**
- * Stop the Nutch server
- * @param force If set to true, it will kill any running jobs
- * @return a message indicating shutdown status
- */
- @GET
- @Path(value="/stop")
- public String stopServer(@QueryParam("force") boolean force){
- if(!server.canStop(force)){
- return "Jobs still running -- Cannot stop server now" ;
- }
- scheduleServerStop();
- return "Stopping in server on port " + server.getPort();
- }
-
- private void scheduleServerStop() {
- LOG.info("Shutting down server in {} sec", DELAY_SEC);
- Thread thread = new Thread() {
- @Override
- public void run() {
- try {
- Thread.sleep(DELAY_SEC*1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- server.stop();
- LOG.info("Service stopped.");
- }
- };
- thread.setDaemon(true);
- thread.start();
- LOG.info("Service shutting down...");
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/resources/ConfigResource.java b/src/java/org/apache/nutch/service/resources/ConfigResource.java
deleted file mode 100644
index 38e14dcf3..000000000
--- a/src/java/org/apache/nutch/service/resources/ConfigResource.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import org.apache.nutch.service.model.request.NutchConfig;
-import com.fasterxml.jackson.jaxrs.annotation.JacksonFeatures;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-@Path("/config")
-public class ConfigResource extends AbstractResource{
-
- public static final String DEFAULT = "default";
-
- /**
- * Returns a list of all configurations created.
- * @return List of configurations
- */
- @GET
- @Path("/")
- @JacksonFeatures(serializationEnable = { SerializationFeature.INDENT_OUTPUT })
- public Set<String> getConfigs() {
- return configManager.list();
- }
-
- /**
- * Get configuration properties
- * @param configId The configuration ID to fetch
- * @return HashMap of the properties set within the given configId
- */
- @GET
- @Path("/{configId}")
- @JacksonFeatures(serializationEnable = { SerializationFeature.INDENT_OUTPUT })
- public Map<String, String> getConfig(@PathParam("configId") String configId) {
- return configManager.getAsMap(configId);
- }
-
- /**
- * Get property
- * @param configId The ID of the configuration
- * @param propertyId The name(key) of the property
- * @return value of the specified property in the provided configId.
- */
- @GET
- @Path("/{configId}/{propertyId}")
- @Produces(MediaType.TEXT_PLAIN)
- @JacksonFeatures(serializationEnable = { SerializationFeature.INDENT_OUTPUT })
- public String getProperty(@PathParam("configId") String configId,
- @PathParam("propertyId") String propertyId) {
- return configManager.getAsMap(configId).get(propertyId);
- }
-
- /**
- * Removes the configuration from the list of known configurations.
- * @param configId The ID of the configuration to delete
- */
- @DELETE
- @Path("/{configId}")
- public void deleteConfig(@PathParam("configId") String configId) {
- configManager.delete(configId);
- }
-
- /**
- * Create new configuration.
- * @param newConfig a new populated {@link NutchConfig}
- * @return The name of the new configuration created
- */
- @POST
- @Path("/create")
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.TEXT_PLAIN)
- public Response createConfig(NutchConfig newConfig) {
- if (newConfig == null) {
- return Response.status(400)
- .entity("Nutch configuration cannot be empty!").build();
- }
- try{
- configManager.create(newConfig);
- }catch(Exception e){
- return Response.status(400)
- .entity(e.getMessage()).build();
- }
- return Response.ok(newConfig.getConfigId()).build();
- }
-
- /**
- * Adds/Updates a particular property value in the configuration
- * @param confId Configuration ID whose property needs to be updated. Make sure that the given
- * confId exists to prevent errors.
- * @param propertyKey Name of the property
- * @param value Value as a simple text
- * @return Success code
- */
- @PUT
- @Path("/{configId}/{propertyId}")
- @Consumes(MediaType.TEXT_PLAIN)
- public Response updateProperty(@PathParam("configId")String confId,
- @PathParam("propertyId")String propertyKey, String value) {
- try{
- configManager.setProperty(confId, propertyKey, value);
- }catch(Exception e) {
- return Response.status(400).entity(e.getMessage()).build();
- }
- return Response.ok().build();
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/DbResource.java b/src/java/org/apache/nutch/service/resources/DbResource.java
deleted file mode 100644
index dc7049a22..000000000
--- a/src/java/org/apache/nutch/service/resources/DbResource.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.crawl.CrawlDbReader;
-import org.apache.nutch.fetcher.FetchNode;
-import org.apache.nutch.fetcher.FetchNodeDb;
-import org.apache.nutch.service.model.request.DbQuery;
-import org.apache.nutch.service.model.response.FetchNodeDbInfo;
-
-@Path(value = "/db")
-public class DbResource extends AbstractResource {
-
- @POST
- @Path(value = "/crawldb")
- @Consumes(MediaType.APPLICATION_JSON)
- public Response readdb(DbQuery dbQuery){
- if(dbQuery == null)
- return Response.status(Status.BAD_REQUEST).build();
-
- Configuration conf = configManager.get(dbQuery.getConfId());
- if(conf == null){
- conf = configManager.get(ConfigResource.DEFAULT);
- }
- if(dbQuery.getCrawlId() == null || dbQuery.getType() == null){
- return Response.status(Status.BAD_REQUEST).build();
- }
- String type = dbQuery.getType();
-
- if(type.equalsIgnoreCase("stats")){
- return crawlDbStats(conf, dbQuery.getArgs(), dbQuery.getCrawlId());
- }
- if(type.equalsIgnoreCase("dump")){
- return crawlDbDump(conf, dbQuery.getArgs(), dbQuery.getCrawlId());
- }
- if(type.equalsIgnoreCase("topN")){
- return crawlDbTopN(conf, dbQuery.getArgs(), dbQuery.getCrawlId());
- }
- if(type.equalsIgnoreCase("url")){
- return crawlDbUrl(conf, dbQuery.getArgs(), dbQuery.getCrawlId());
- }
- return null;
-
- }
-
- @GET
- @Path(value="/fetchdb")
- public List<FetchNodeDbInfo> fetchDb(@DefaultValue("0")@QueryParam("to")int to, @DefaultValue("0")@QueryParam("from")int from){
- List<FetchNodeDbInfo> listOfFetchedNodes = new ArrayList<>();
- Map<Integer, FetchNode> fetchNodedbMap = FetchNodeDb.getInstance().getFetchNodeDb();
-
- if(to ==0 || to>fetchNodedbMap.size()){
- to = fetchNodedbMap.size();
- }
- for(int i=from;i<=to;i++){
- if(!fetchNodedbMap.containsKey(i)){
- continue;
- }
- FetchNode node = fetchNodedbMap.get(i);
- FetchNodeDbInfo fdbInfo = new FetchNodeDbInfo();
- fdbInfo.setUrl(node.getUrl().toString());
- fdbInfo.setStatus(node.getStatus());
- fdbInfo.setNumOfOutlinks(node.getOutlinks().length);
- fdbInfo.setChildNodes(node.getOutlinks());
- listOfFetchedNodes.add(fdbInfo);
- }
-
- return listOfFetchedNodes;
- }
- @SuppressWarnings("resource")
- private Response crawlDbStats(Configuration conf, Map<String, String> args, String crawlId){
- CrawlDbReader dbr = new CrawlDbReader();
- try{
- return Response.ok(dbr.query(args, conf, "stats", crawlId)).build();
- }catch(Exception e){
- e.printStackTrace();
- return Response.serverError().entity(e.getMessage()).type(MediaType.TEXT_PLAIN).build();
- }
- }
-
- @Produces(MediaType.APPLICATION_OCTET_STREAM)
- private Response crawlDbDump(Configuration conf, Map<String, String> args, String crawlId){
- @SuppressWarnings("resource")
- CrawlDbReader dbr = new CrawlDbReader();
- try{
- return Response.ok(dbr.query(args, conf, "dump", crawlId), MediaType.APPLICATION_OCTET_STREAM).build();
- }catch(Exception e){
- e.printStackTrace();
- return Response.serverError().entity(e.getMessage()).type(MediaType.TEXT_PLAIN).build();
- }
- }
-
- @Produces(MediaType.APPLICATION_OCTET_STREAM)
- private Response crawlDbTopN(Configuration conf, Map<String, String> args, String crawlId) {
- @SuppressWarnings("resource")
- CrawlDbReader dbr = new CrawlDbReader();
- try{
- return Response.ok(dbr.query(args, conf, "topN", crawlId), MediaType.APPLICATION_OCTET_STREAM).build();
- }catch(Exception e){
- e.printStackTrace();
- return Response.serverError().entity(e.getMessage()).type(MediaType.TEXT_PLAIN).build();
- }
- }
-
- private Response crawlDbUrl(Configuration conf, Map<String, String> args, String crawlId){
- @SuppressWarnings("resource")
- CrawlDbReader dbr = new CrawlDbReader();
- try{
- return Response.ok(dbr.query(args, conf, "url", crawlId)).build();
- }catch(Exception e){
- e.printStackTrace();
- return Response.serverError().entity(e.getMessage()).type(MediaType.TEXT_PLAIN).build();
- }
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/JobResource.java b/src/java/org/apache/nutch/service/resources/JobResource.java
deleted file mode 100644
index 0641d2160..000000000
--- a/src/java/org/apache/nutch/service/resources/JobResource.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.util.Collection;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.jaxrs.annotation.JacksonFeatures;
-import org.apache.nutch.service.model.request.JobConfig;
-import org.apache.nutch.service.model.response.JobInfo;
-import org.apache.nutch.service.model.response.JobInfo.State;
-
-@Path(value = "/job")
-public class JobResource extends AbstractResource {
-
- /**
- * Get job history for a given job regardless of the jobs state
- * @param crawlId a crawlId
- * @return A nested JSON object of all the jobs created for that crawlId
- */
- @GET
- @Path(value = "/")
- @JacksonFeatures(serializationEnable = { SerializationFeature.INDENT_OUTPUT })
- public Collection<JobInfo> getJobs(@QueryParam("crawlId") String crawlId) {
- return jobManager.list(crawlId, State.ANY);
- }
-
- /**
- * Get job info
- * @param id Job ID
- * @param crawlId Crawl ID
- * @return A JSON object of job parameters
- */
- @GET
- @Path(value = "/{id}")
- @JacksonFeatures(serializationEnable = { SerializationFeature.INDENT_OUTPUT })
- public JobInfo getInfo(@PathParam("id") String id,
- @QueryParam("crawlId") String crawlId) {
- return jobManager.get(crawlId, id);
- }
-
- /**
- * Stop Job
- * @param id Job ID
- * @param crawlId Crawl ID
- * @return true if stopped, false otherwise
- */
- @GET
- @Path(value = "/{id}/stop")
- public boolean stop(@PathParam("id") String id,
- @QueryParam("crawlId") String crawlId) {
- return jobManager.stop(crawlId, id);
- }
-
-
- @GET
- @Path(value = "/{id}/abort")
- public boolean abort(@PathParam("id") String id,
- @QueryParam("crawlId") String crawlId) {
- return jobManager.abort(crawlId, id);
- }
-
- /**
- * Create a new job
- * @param config The parameters of the job to create
- * @return A JSON object of the job created with its details
- */
- @POST
- @Path(value = "/create")
- @Consumes(MediaType.APPLICATION_JSON)
- public JobInfo create(JobConfig config) {
- if (config == null) {
- throwBadRequestException("Job configuration is required!");
- }
- return jobManager.create(config);
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/ReaderResouce.java b/src/java/org/apache/nutch/service/resources/ReaderResouce.java
deleted file mode 100644
index f2f52e9c2..000000000
--- a/src/java/org/apache/nutch/service/resources/ReaderResouce.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.util.HashMap;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.nutch.service.NutchReader;
-import org.apache.nutch.service.impl.LinkReader;
-import org.apache.nutch.service.impl.NodeReader;
-import org.apache.nutch.service.impl.SequenceReader;
-import org.apache.nutch.service.model.request.ReaderConfig;
-
-/**
- * The Reader endpoint enables a user to read sequence files,
- * nodes and links from the Nutch webgraph.
- * @author Sujen Shah
- *
- */
-@Path("/reader")
-public class ReaderResouce {
-
- /**
- * Read a sequence file
- * @param readerConf A initialized {@link ReaderConfig}
- * @param nrows Number of rows to read. If not specified all rows will be read
- * @param start Specify a starting line number to read the file from
- * @param end The line number to read the file till
- * @param count Boolean value. If true, this endpoint will return the number of lines in the line
- * @return Appropriate HTTP response based on the query
- */
- @Path("/sequence/read")
- @POST
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response seqRead(ReaderConfig readerConf,
- @DefaultValue("-1")@QueryParam("nrows") int nrows,
- @DefaultValue("-1")@QueryParam("start") int start,
- @QueryParam("end")int end, @QueryParam("count") boolean count) {
-
- NutchReader reader = new SequenceReader();
- String path = readerConf.getPath();
- return performRead(reader, path, nrows, start, end, count);
- }
-
- /**
- * Get Link Reader response schema
- * @return JSON object specifying the schema of the responses returned by the Link Reader
- */
- @Path("/link")
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response linkRead() {
- HashMap<String, String> schema = new HashMap<>();
- schema.put("key_url","string");
- schema.put("timestamp", "int");
- schema.put("score","float");
- schema.put("anchor","string");
- schema.put("linktype","string");
- schema.put("url","string");
- return Response.ok(schema).type(MediaType.APPLICATION_JSON).build();
- }
-
- /**
- * Read link object
- * @param readerConf A initialized {@link ReaderConfig}
- * @param nrows Number of rows to read. If not specified all rows will be read
- * @param start Specify a starting line number to read the file from
- * @param end The line number to read the file till
- * @param count Boolean value. If true, this endpoint will return the number of lines in the line
- * @return Appropriate HTTP response based on the query
- */
- @Path("/link/read")
- @POST
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response linkRead(ReaderConfig readerConf,
- @DefaultValue("-1")@QueryParam("nrows") int nrows,
- @DefaultValue("-1")@QueryParam("start") int start,
- @QueryParam("end") int end, @QueryParam("count") boolean count) {
-
- NutchReader reader = new LinkReader();
- String path = readerConf.getPath();
- return performRead(reader, path, nrows, start, end, count);
- }
-
- /**
- * Get schema of the Node object
- * @return JSON object specifying the schema of the responses returned by the Node Reader
- */
- @Path("/node")
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response nodeRead() {
- HashMap<String, String> schema = new HashMap<>();
- schema.put("key_url","string");
- schema.put("num_inlinks", "int");
- schema.put("num_outlinks","int");
- schema.put("inlink_score","float");
- schema.put("outlink_score","float");
- schema.put("metadata","string");
- return Response.ok(schema).type(MediaType.APPLICATION_JSON).build();
- }
-
-
- /**
- * Read Node object as stored in the Nutch Webgraph
- * @param readerConf A initialized {@link ReaderConfig}
- * @param nrows Number of rows to read. If not specified all rows will be read
- * @param start Specify a starting line number to read the file from
- * @param end The line number to read the file till
- * @param count Boolean value. If true, this endpoint will return the number of lines in the line
- * @return Appropriate HTTP response based on the query
- */
- @Path("/node/read")
- @POST
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response nodeRead(ReaderConfig readerConf,
- @DefaultValue("-1")@QueryParam("nrows") int nrows,
- @DefaultValue("-1")@QueryParam("start") int start,
- @QueryParam("end") int end, @QueryParam("count") boolean count) {
-
- NutchReader reader = new NodeReader();
- String path = readerConf.getPath();
- return performRead(reader, path, nrows, start, end, count);
- }
-
-
- private Response performRead(NutchReader reader, String path,
- int nrows, int start, int end, boolean count) {
- Object result;
- try{
- if(count){
- result = reader.count(path);
- return Response.ok(result).type(MediaType.TEXT_PLAIN).build();
- }
- else if(start>-1 && end>0) {
- result = reader.slice(path, start, end);
- }
- else if(nrows>-1) {
- result = reader.head(path, nrows);
- }
- else {
- result = reader.read(path);
- }
- return Response.ok(result).type(MediaType.APPLICATION_JSON).build();
- }catch(Exception e){
- return Response.status(Status.BAD_REQUEST).entity("File not found").build();
- }
- }
-
-}
diff --git a/src/java/org/apache/nutch/service/resources/SeedResource.java b/src/java/org/apache/nutch/service/resources/SeedResource.java
deleted file mode 100644
index a1a555141..000000000
--- a/src/java/org/apache/nutch/service/resources/SeedResource.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.io.OutputStream;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Map;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.nutch.service.NutchServer;
-import org.apache.nutch.service.model.request.SeedList;
-import org.apache.nutch.service.model.request.SeedUrl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Path("/seed")
-public class SeedResource extends AbstractResource {
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
-
- /**
- * Gets the list of seedFiles already created
- * @return {@link Map} of <String, SeedList>
- */
- @GET
- @Path("/")
- @Produces(MediaType.APPLICATION_JSON)
- public Response getSeedLists() {
- Map<String, SeedList> seeds = NutchServer.getInstance().getSeedManager().getSeeds();
- if(seeds!=null) {
- return Response.ok(seeds).build();
- }
- else {
- return Response.ok().build();
- }
- }
-
- /**
- * Method creates seed list file and returns temporary directory path
- * @param seedList a populated {@link SeedList}
- * @return path to seedfile
- */
- @POST
- @Path("/create")
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.TEXT_PLAIN)
- public Response createSeedFile(SeedList seedList) {
- try {
- if (seedList == null) {
- return Response.status(Status.BAD_REQUEST)
- .entity("Seed list cannot be empty!").build();
- }
- Collection<SeedUrl> seedUrls = seedList.getSeedUrls();
-
- String seedFilePath = writeToSeedFile(seedUrls);
- seedList.setSeedFilePath(seedFilePath);
- NutchServer.getInstance().getSeedManager().
- setSeedList(seedList.getName(), seedList);
- return Response.ok().entity(seedFilePath).build();
- } catch (Exception e) {
- LOG.warn("Error while creating seed : {}", e.getMessage());
- }
- return Response.serverError().build();
- }
-
- private String writeToSeedFile(Collection<SeedUrl> seedUrls) throws Exception {
- String seedFilePath = "seedFiles/seed-" + System.currentTimeMillis();
- org.apache.hadoop.fs.Path seedFolder = new org.apache.hadoop.fs.Path(seedFilePath);
- FileSystem fs = FileSystem.get(new Configuration());
- if(!fs.exists(seedFolder)) {
- if(!fs.mkdirs(seedFolder)) {
- throw new Exception("Could not create seed folder at : " + seedFolder);
- }
- }
- String filename = seedFilePath + System.getProperty("file.separator") + "urls";
- org.apache.hadoop.fs.Path seedPath = new org.apache.hadoop.fs.Path(filename);
- OutputStream os = fs.create(seedPath);
- if (CollectionUtils.isNotEmpty(seedUrls)) {
- for (SeedUrl seedUrl : seedUrls) {
- os.write(seedUrl.getUrl().getBytes());
- os.write("\n".getBytes());
- }
- }
- os.close();
- return seedPath.getParent().toString();
- }
-}
diff --git a/src/java/org/apache/nutch/service/resources/ServicesResource.java b/src/java/org/apache/nutch/service/resources/ServicesResource.java
deleted file mode 100644
index c129652c3..000000000
--- a/src/java/org/apache/nutch/service/resources/ServicesResource.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service.resources;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import org.apache.nutch.metadata.Nutch;
-import org.apache.nutch.service.impl.ServiceWorker;
-import org.apache.nutch.service.model.request.ServiceConfig;
-import org.apache.nutch.service.model.response.ServiceInfo;
-import org.apache.nutch.tools.CommonCrawlDataDumper;
-
-/**
- * The services resource defines an endpoint to enable the user to carry out
- * Nutch jobs like dump, commoncrawldump, etc.
- */
-@Path("/services")
-public class ServicesResource {
-
- private static SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
-
- @GET
- @Path("/commoncrawldump/{crawlId}")
- public Response listDumpPaths(@PathParam("crawlId") String crawlId) {
- File dumpFilePath = new File(crawlId + File.separator + "dump/");
- File dumpFileList[] = dumpFilePath.listFiles();
- List<String> fileNames = new ArrayList<>();
- if (dumpFileList != null) {
- for (File f : dumpFileList) {
- fileNames.add(f.getPath());
- }
- }
- ServiceInfo info = new ServiceInfo();
- info.setDumpPaths(fileNames);
- return Response.ok().entity(info).type(MediaType.APPLICATION_JSON).build();
- }
-
- @POST
- @Path("/commoncrawldump")
- public Response commoncrawlDump(ServiceConfig serviceConfig) {
- String crawlId = serviceConfig.getCrawlId();
- String outputDir = crawlId + File.separator + "dump" + File.separator
- + "commoncrawl-" + sdf.format(System.currentTimeMillis());
-
- Map<String, Object> args = serviceConfig.getArgs();
- args.put("outputDir", outputDir);
- if (!args.containsKey(Nutch.ARG_SEGMENTDIR)) {
- args.put("segment", crawlId + File.separator + "segments");
- }
- serviceConfig.setArgs(args);
- ServiceWorker worker = new ServiceWorker(serviceConfig,
- new CommonCrawlDataDumper());
- worker.run();
-
- return Response.ok(outputDir).type(MediaType.TEXT_PLAIN).build();
- }
-
-}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index 8e37c21fc..0b6d48e53 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -722,7 +722,7 @@ public class CommonCrawlDataDumper extends NutchTool implements Tool {
}
/**
- * Used by the REST service
+ * Programmatic entry point for {@link org.apache.nutch.util.NutchTool} callers.
*/
@Override
public Map<String, Object> run(Map<String, Object> args, String crawlId)
diff --git a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
index 581b528f3..d11634bbd 100644
--- a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
+++ b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
@@ -18,11 +18,16 @@ package org.apache.nutch.crawl;
import java.lang.invoke.MethodHandles;
import java.io.IOException;
-import java.net.UnknownHostException;
import java.net.URI;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.mockserver.model.HttpRequest.request;
+import static org.mockserver.model.HttpResponse.response;
import org.jspecify.annotations.NonNull;
import org.jspecify.annotations.Nullable;
@@ -50,10 +55,8 @@ import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.Reducer.Context;
import org.apache.hadoop.security.Credentials;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.handler.ResourceHandler;
+import org.mockserver.configuration.ConfigurationProperties;
+import org.mockserver.integration.ClientAndServer;
/**
* Test utility for creating and manipulating CrawlDb instances.
@@ -452,29 +455,49 @@ public class CrawlDBTestUtil {
}
/**
- * Creates a new JettyServer with one static root context
- *
+ * Starts a <a href="https://www.mock-server.com/">MockServer</a> instance that
+ * serves files from {@code staticContent} for GET requests (one expectation per
+ * file).
+ *
* @param port
- * port to listen to
+ * port to listen on
* @param staticContent
- * folder where static content lives
- * @return configured Jetty server instance
- * @throws UnknownHostException
+ * directory of static files
+ * @return running mock server (already started); call {@link ClientAndServer#stop()} when done
*/
@NonNull
- public static Server getServer(int port, @NonNull String staticContent)
- throws UnknownHostException {
- Server webServer = new Server();
-
- ServerConnector listener = new ServerConnector(webServer);
- listener.setPort(port);
- listener.setHost("127.0.0.1");
- webServer.addConnector(listener);
- ContextHandler staticContext = new ContextHandler();
- staticContext.setContextPath("/");
- staticContext.setResourceBase(staticContent);
- staticContext.insertHandler(new ResourceHandler());
- webServer.insertHandler(staticContext);
- return webServer;
+ public static ClientAndServer startMockServerForStaticContent(int port,
+ @NonNull String staticContent) throws IOException {
+ ConfigurationProperties.disableLogging(true);
+ java.nio.file.Path root = java.nio.file.Path.of(staticContent)
+ .toAbsolutePath().normalize();
+ ClientAndServer mockServer = ClientAndServer.startClientAndServer(port);
+ java.nio.file.Path indexPath = root.resolve("index.html");
+ if (Files.isRegularFile(indexPath)) {
+ byte[] data = Files.readAllBytes(indexPath);
+ mockServer.when(request().withMethod("GET").withPath("/"))
+ .respond(response().withStatusCode(200)
+ .withHeader("Content-Type", probeContentType(indexPath))
+ .withBody(data));
+ }
+ try (Stream<java.nio.file.Path> walk = Files.walk(root)) {
+ for (java.nio.file.Path file : walk.filter(Files::isRegularFile)
+ .collect(Collectors.toList())) {
+ String rel = root.relativize(file).toString().replace('\\', '/');
+ String mockPath = "/" + rel;
+ byte[] data = Files.readAllBytes(file);
+ mockServer.when(request().withMethod("GET").withPath(mockPath))
+ .respond(response().withStatusCode(200)
+ .withHeader("Content-Type", probeContentType(file))
+ .withBody(data));
+ }
+ }
+ return mockServer;
+ }
+
+ private static String probeContentType(java.nio.file.Path file)
+ throws IOException {
+ String ct = Files.probeContentType(file);
+ return ct != null ? ct : "application/octet-stream";
}
}
diff --git a/src/test/org/apache/nutch/fetcher/TestFetcher.java b/src/test/org/apache/nutch/fetcher/TestFetcher.java
index 176a88a52..e7030bfe5 100644
--- a/src/test/org/apache/nutch/fetcher/TestFetcher.java
+++ b/src/test/org/apache/nutch/fetcher/TestFetcher.java
@@ -30,7 +30,7 @@ import org.apache.nutch.parse.ParseData;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.util.CancellationAwareTestUtils;
import org.apache.nutch.util.CancellationAwareTestUtils.CancellationToken;
-import org.eclipse.jetty.server.Server;
+import org.mockserver.integration.ClientAndServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -59,7 +59,7 @@ public class TestFetcher {
Path crawldbPath;
Path segmentsPath;
Path urlPath;
- Server server;
+ ClientAndServer mockServer;
@BeforeEach
public void setUp() throws Exception {
@@ -69,19 +69,15 @@ public class TestFetcher {
urlPath = new Path(testdir, "urls");
crawldbPath = new Path(testdir, "crawldb");
segmentsPath = new Path(testdir, "segments");
- server = CrawlDBTestUtil.getServer(
+ mockServer = CrawlDBTestUtil.startMockServerForStaticContent(
conf.getInt("content.server.port", 50000),
"build/test/data/fetch-test-site");
- server.start();
}
@AfterEach
public void tearDown() throws Exception {
- server.stop();
- for (int i = 0; i < 5; i++) {
- if (!server.isStopped()) {
- Thread.sleep(1000);
- }
+ if (mockServer != null) {
+ mockServer.stop();
}
fs.delete(testdir, true);
}
@@ -224,7 +220,7 @@ public class TestFetcher {
}
private void addUrl(ArrayList<String> urls, String page) {
- urls.add("http://127.0.0.1:" + server.getURI().getPort() + "/" + page);
+ urls.add("http://127.0.0.1:" + mockServer.getLocalPort() + "/" + page);
}
@Test
diff --git a/src/test/org/apache/nutch/service/TestNutchServer.java b/src/test/org/apache/nutch/service/TestNutchServer.java
deleted file mode 100644
index 11397a9b1..000000000
--- a/src/test/org/apache/nutch/service/TestNutchServer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.service;
-
-import java.lang.invoke.MethodHandles;
-import javax.ws.rs.core.Response;
-
-import org.apache.cxf.jaxrs.client.WebClient;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestNutchServer {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(MethodHandles.lookup().lookupClass());
- NutchServer server = NutchServer.getInstance();
-
- private int port[] = {8081, 9999, 9100, 8900};
- private final String ENDPOINT_ADDRESS = "http://localhost:";
-
- @Test
- public void testNutchServerStartup() {
- boolean isRunning = false;
- for(int i=0;i<port.length; i++) {
- try {
- startServer(port[i]);
- isRunning = true;
- break;
- }catch(Exception e) {
- LOG.info("Could not start server on port: {}. Tries remaining {}", port[i], port.length-i);
- }
- }
- if(!isRunning) {
- LOG.info("Could not start server, all ports in use");
- }
- else {
- LOG.info("Testing admin endpoint");
- WebClient client = WebClient.create(ENDPOINT_ADDRESS + server.getPort());
- @SuppressWarnings("unused")
- Response response = client.path("admin").get();
- //Assert.assertTrue(response.readEntity(String.class).contains("startDate"));
- response = client.path("stop").get();
- //Assert.assertTrue(response.readEntity(String.class).contains("Stopping"));
- }
- }
-
- private void startServer(int port) throws Exception{
- NutchServer.setPort(port);
- NutchServer.startServer();
- }
-}