You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2018/06/01 08:24:07 UTC
asterixdb git commit: [NO ISSUE][ING] Allow Readers to add their own
stats
Repository: asterixdb
Updated Branches:
refs/heads/master bc7e16e05 -> dbae78f5a
[NO ISSUE][ING] Allow Readers to add their own stats
- user model changes: no
- storage format changes: no
- interface changes: yes
- Add IRecordReader.getStats()
Details:
- Allow record reader to contribute their own stats.
Change-Id: Ib064bd97b33f2d9da330bac3a49bd4c6831faaca
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2642
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/dbae78f5
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/dbae78f5
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/dbae78f5
Branch: refs/heads/master
Commit: dbae78f5a0a412bb95158627b2fe7e17488b233c
Parents: bc7e16e
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Thu May 31 23:47:00 2018 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Fri Jun 1 01:23:34 2018 -0700
----------------------------------------------------------------------
.../active/ActiveSourceOperatorNodePushable.java | 4 +++-
.../org/apache/asterix/external/api/IRecordReader.java | 7 +++++++
.../dataflow/FeedRecordDataFlowController.java | 13 +++++++++++--
3 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
index 1df9020..2f30df4 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
@@ -63,8 +63,10 @@ public abstract class ActiveSourceOperatorNodePushable extends AbstractUnaryOutp
@Override
public final void stop() throws HyracksDataException, InterruptedException {
- abort();
synchronized (this) {
+ if (!done) {
+ abort();
+ }
while (!done) {
wait();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index 08ffe18..9033814 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -77,4 +77,11 @@ public interface IRecordReader<T> extends Closeable {
public default IFeedMarker getProgressReporter() {
return null;
}
+
+ /**
+ * @return JSON String containing ingestion stats
+ */
+ default String getStats() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 164ff68..f392139 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -40,6 +40,7 @@ import org.apache.logging.log4j.Logger;
public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowController {
public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = "incoming-records-count";
public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = "failed-at-parser-records-count";
+ public static final String READER_STATS_FIELD_NAME = "reader-stats";
public enum State {
CREATED,
@@ -257,7 +258,15 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
@Override
public String getStats() {
- return "{\"" + INCOMING_RECORDS_COUNT_FIELD_NAME + "\": " + incomingRecordsCount + ", \""
- + FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME + "\": " + failedRecordsCount + "}";
+ String readerStats = recordReader.getStats();
+ StringBuilder str = new StringBuilder();
+ str.append("{");
+ if (readerStats != null) {
+ str.append("\"").append(READER_STATS_FIELD_NAME).append("\":").append(readerStats).append(", ");
+ }
+ str.append("\"").append(INCOMING_RECORDS_COUNT_FIELD_NAME).append("\": ").append(incomingRecordsCount)
+ .append(", \"").append(FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME).append("\": ")
+ .append(failedRecordsCount).append("}");
+ return str.toString();
}
}