You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/05/02 16:48:24 UTC
Change in asterixdb[trinity]: [NO ISSUE]: Track more requests + jobs counts
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18263 )
Change subject: [NO ISSUE]: Track more requests + jobs counts
......................................................................
[NO ISSUE]: Track more requests + jobs counts
Change-Id: I8fa31a1e6bb6b1f1bcf90c59da646fc47546fc7c
---
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestCapacityController.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
5 files changed, 129 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/63/18263/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index f84e45a..07c76a0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -433,7 +433,7 @@
return true;
case REQUEST_TIMEOUT:
LOGGER.info(() -> "handleException: request execution timed out: " + param.toString());
- executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
+ executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.SERVICE_UNAVAILABLE);
return true;
case REJECT_NODE_UNREGISTERED:
case REJECT_BAD_CLUSTER_STATE:
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
index b2795d4..8d44542 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
@@ -130,4 +130,24 @@
* @return the maximum number of jobs to queue before rejecting new jobs
*/
int getJobQueueCapacity();
+
+ /**
+ * @return total jobs
+ */
+ long getTotalJobs();
+
+ /**
+ * @return total failed jobs
+ */
+ long getTotalFailedJobs();
+
+ /**
+ * @return total cancelled jobs
+ */
+ long getTotalCancelledJobs();
+
+ /**
+ * @return total rejected jobs
+ */
+ long getTotalRejectedJobs();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index 4882f4a..7a99a3d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -30,10 +30,12 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IError;
import org.apache.hyracks.api.job.ActivityClusterGraph;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -66,11 +68,19 @@
private final Map<JobId, JobRun> runMapArchive;
private final Map<JobId, List<Exception>> runMapHistory;
private final IJobCapacityController jobCapacityController;
+ private final AtomicLong totalJobs;
+ private final AtomicLong totalFailedJobs;
+ private final AtomicLong totalCancelledJobs;
+ private final AtomicLong totalRejectedJobs;
private IJobQueue jobQueue;
public JobManager(CCConfig ccConfig, ClusterControllerService ccs, IJobCapacityController jobCapacityController) {
this.ccs = ccs;
this.jobCapacityController = jobCapacityController;
+ this.totalJobs = new AtomicLong();
+ this.totalFailedJobs = new AtomicLong();
+ this.totalCancelledJobs = new AtomicLong();
+ this.totalRejectedJobs = new AtomicLong();
try {
Constructor<?> jobQueueConstructor = this.getClass().getClassLoader().loadClass(ccConfig.getJobQueueClass())
.getConstructor(IJobManager.class, IJobCapacityController.class);
@@ -84,7 +94,7 @@
jobQueue = new FIFOJobQueue(this, jobCapacityController);
}
activeRunMap = new HashMap<>();
- runMapArchive = new LinkedHashMap<JobId, JobRun>() {
+ runMapArchive = new LinkedHashMap<>() {
private static final long serialVersionUID = -1406441385508773629L;
@Override
@@ -92,7 +102,7 @@
return size() > ccConfig.getJobHistorySize();
}
};
- runMapHistory = new LinkedHashMap<JobId, List<Exception>>() {
+ runMapHistory = new LinkedHashMap<>() {
private static final long serialVersionUID = 7572062687032652986L;
/** history size + 1 is for the case when history size = 0 */
private final int allowedSize = 100 * (ccConfig.getJobHistorySize() + 1);
@@ -108,18 +118,25 @@
public void add(JobRun jobRun) throws HyracksException {
checkJob(jobRun);
JobSpecification job = jobRun.getJobSpecification();
- IJobCapacityController.JobSubmissionStatus status = jobCapacityController.allocate(job);
- CCServiceContext serviceCtx = ccs.getContext();
- serviceCtx.notifyJobCreation(jobRun.getJobId(), job);
- switch (status) {
- case QUEUE:
- queueJob(jobRun);
- break;
- case EXECUTE:
- executeJob(jobRun);
- break;
- default:
- throw new IllegalStateException("unknown submission status: " + status);
+ IJobCapacityController.JobSubmissionStatus status;
+ try {
+ status = jobCapacityController.allocate(job);
+ incrementTotalJobs();
+ CCServiceContext serviceCtx = ccs.getContext();
+ serviceCtx.notifyJobCreation(jobRun.getJobId(), job);
+ switch (status) {
+ case QUEUE:
+ queueJob(jobRun);
+ break;
+ case EXECUTE:
+ executeJob(jobRun);
+ break;
+ default:
+ throw new IllegalStateException("unknown submission status: " + status);
+ }
+ } catch (HyracksDataException ex) {
+ handleException(ex);
+ throw ex;
}
}
@@ -132,6 +149,7 @@
// trigger JobCleanupWork and JobCleanupNotificationWork which will update the lifecyle of the job.
// Therefore, we do not remove the job out of activeRunMap here.
jobRun.getExecutor().cancelJob(callback);
+ incrementTotalCancelledJobs();
return;
}
// Removes a pending job.
@@ -219,6 +237,10 @@
@Override
public void finalComplete(JobRun run) throws HyracksException {
checkJob(run);
+ if (run.getPendingStatus() == JobStatus.FAILURE) {
+ incrementTotalFailedJobs();
+ }
+
JobId jobId = run.getJobId();
Throwable caughtException = null;
CCServiceContext serviceCtx = ccs.getContext();
@@ -301,6 +323,26 @@
return ccs.getCCConfig().getJobQueueCapacity();
}
+ @Override
+ public long getTotalJobs() {
+ return totalJobs.get();
+ }
+
+ @Override
+ public long getTotalFailedJobs() {
+ return totalFailedJobs.get();
+ }
+
+ @Override
+ public long getTotalCancelledJobs() {
+ return totalCancelledJobs.get();
+ }
+
+ @Override
+ public long getTotalRejectedJobs() {
+ return totalRejectedJobs.get();
+ }
+
private void pickJobsToRun() throws HyracksException {
List<JobRun> selectedRuns = jobQueue.pull();
for (JobRun run : selectedRuns) {
@@ -356,4 +398,31 @@
final JobSpecification job = jobRun.getJobSpecification();
jobCapacityController.release(job);
}
+
+ private void handleException(HyracksException ex) {
+ if (ex.getError().isPresent()) {
+ IError error = ex.getError().get();
+ switch ((ErrorCode) error) {
+ case JOB_QUEUE_FULL:
+ case JOB_REQUIREMENTS_EXCEED_CAPACITY:
+ incrementTotalRejectedJobs();
+ }
+ }
+ }
+
+ private void incrementTotalJobs() {
+ totalJobs.incrementAndGet();
+ }
+
+ private void incrementTotalFailedJobs() {
+ totalFailedJobs.incrementAndGet();
+ }
+
+ private void incrementTotalCancelledJobs() {
+ totalCancelledJobs.incrementAndGet();
+ }
+
+ private void incrementTotalRejectedJobs() {
+ totalRejectedJobs.incrementAndGet();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestCapacityController.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestCapacityController.java
index 3ab2ab9..351365c 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestCapacityController.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestCapacityController.java
@@ -52,13 +52,14 @@
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (overloaded) {
+ incrementRejectedRequestsCount();
ReferenceCountUtil.release(msg);
return;
}
if (overloaded()) {
ReferenceCountUtil.release(msg);
+ incrementRejectedRequestsCount();
reject(ctx);
- return;
} else {
super.channelRead(ctx, msg);
}
@@ -85,6 +86,7 @@
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
if (overloaded()) {
+ incrementRejectedRequestsCount();
reject(ctx);
return;
}
@@ -105,4 +107,8 @@
}
return overloaded;
}
+
+ private void incrementRejectedRequestsCount() {
+ server.incrementRejectedRequestsCount();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 271de53..9fec606 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -87,6 +87,7 @@
private final EventLoopGroup workerGroup;
private final Set<InetSocketAddress> addresses;
private final ThreadPoolExecutor executor;
+ private final AtomicInteger rejectedRequestsCount = new AtomicInteger();
// Mutable members
private volatile int state = STOPPED;
private volatile Thread recoveryThread;
@@ -443,4 +444,12 @@
channels.clear();
}
}
+
+ public void incrementRejectedRequestsCount() {
+ rejectedRequestsCount.incrementAndGet();
+ }
+
+ public int getRejectedRequestsCount() {
+ return rejectedRequestsCount.get();
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18263
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: trinity
Gerrit-Change-Id: I8fa31a1e6bb6b1f1bcf90c59da646fc47546fc7c
Gerrit-Change-Number: 18263
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange