You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2015/04/22 05:37:24 UTC
svn commit: r1675249 - in /nutch/trunk/src/java/org/apache/nutch:
service/model/request/JobConfig.java util/NutchTool.java
Author: mattmann
Date: Wed Apr 22 03:37:24 2015
New Revision: 1675249
URL: http://svn.apache.org/r1675249
Log:
Remainder of fix for NUTCH-1973: Job Administration end point for the REST service contributed by Sujen Shah <su...@gmail.com>
Added:
nutch/trunk/src/java/org/apache/nutch/service/model/request/JobConfig.java
nutch/trunk/src/java/org/apache/nutch/util/NutchTool.java
Added: nutch/trunk/src/java/org/apache/nutch/service/model/request/JobConfig.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/request/JobConfig.java?rev=1675249&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/model/request/JobConfig.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/model/request/JobConfig.java Wed Apr 22 03:37:24 2015
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.service.model.request;
+
+import java.util.Map;
+
+import org.apache.nutch.service.JobManager.JobType;
+
+
+public class JobConfig {
+ private String crawlId;
+ private JobType type;
+ private String confId;
+ private String jobClassName;
+ private Map<String, String> args;
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public JobType getType() {
+ return type;
+ }
+
+ public void setType(JobType type) {
+ this.type = type;
+ }
+
+ public String getConfId() {
+ return confId;
+ }
+
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+
+ public Map<String, String> getArgs() {
+ return args;
+ }
+
+ public void setArgs(Map<String, String> args) {
+ this.args = args;
+ }
+
+ public String getJobClassName() {
+ return jobClassName;
+ }
+
+ public void setJobClassName(String jobClass) {
+ this.jobClassName = jobClass;
+ }
+}
\ No newline at end of file
Added: nutch/trunk/src/java/org/apache/nutch/util/NutchTool.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/NutchTool.java?rev=1675249&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/NutchTool.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/util/NutchTool.java Wed Apr 22 03:37:24 2015
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.util;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.nutch.metadata.Nutch;
+
+public abstract class NutchTool extends Configured {
+
+ protected HashMap<String, Object> results = new HashMap<String, Object>();
+ protected Map<String, Object> status = Collections
+ .synchronizedMap(new HashMap<String, Object>());
+ protected Job currentJob;
+ protected int numJobs;
+ protected int currentJobNum;
+
+ /**
+ * Runs the tool, using a map of arguments. May return results, or null.
+ */
+ public abstract Map<String, Object> run(Map<String, String> args, String crawlId)
+ throws Exception;
+
+ public NutchTool(Configuration conf){
+ super(conf);
+ }
+
+ public NutchTool(){
+ super(null);
+ }
+
+ /** Returns relative progress of the tool, a float in range [0,1]. */
+ public float getProgress() {
+ float res = 0;
+ if (currentJob != null) {
+ try {
+ res = (currentJob.mapProgress() + currentJob.reduceProgress()) / 2.0f;
+ } catch (IOException e) {
+ e.printStackTrace();
+ res = 0;
+ } catch (IllegalStateException ile) {
+ ile.printStackTrace();
+ res = 0;
+ }
+ }
+ // take into account multiple jobs
+ if (numJobs > 1) {
+ res = (currentJobNum + res) / (float) numJobs;
+ }
+ status.put(Nutch.STAT_PROGRESS, res);
+ return res;
+ }
+
+ /** Returns current status of the running tool. */
+ public Map<String, Object> getStatus() {
+ return status;
+ }
+
+ /**
+ * Stop the job with the possibility to resume. Subclasses should override
+ * this, since by default it calls {@link #killJob()}.
+ *
+ * @return true if succeeded, false otherwise
+ */
+ public boolean stopJob() throws Exception {
+ return killJob();
+ }
+
+ /**
+ * Kill the job immediately. Clients should assume that any results that the
+ * job produced so far are in inconsistent state or missing.
+ *
+ * @return true if succeeded, false otherwise.
+ * @throws Exception
+ */
+ public boolean killJob() throws Exception {
+ if (currentJob != null && !currentJob.isComplete()) {
+ try {
+ currentJob.killJob();
+ return true;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file