You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2015/12/18 16:21:56 UTC
nifi git commit: NIFI-108: - Including queue size statistics in
listing request. - Showing connection name.
Repository: nifi
Updated Branches:
refs/heads/NIFI-108 f3385a853 -> ba54e8e3f
NIFI-108:
- Including queue size statistics in listing request.
- Showing connection name.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ba54e8e3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ba54e8e3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ba54e8e3
Branch: refs/heads/NIFI-108
Commit: ba54e8e3fbe4fcd386cc4d875802f93adbe2cbc5
Parents: f3385a8
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Dec 18 10:21:43 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Dec 18 10:21:43 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/web/api/dto/FlowFileDTO.java | 2 +
.../nifi/web/api/dto/FlowFileSummaryDTO.java | 3 ++
.../nifi/web/api/dto/ListingRequestDTO.java | 15 ++++++
.../cluster/manager/impl/WebClusterManager.java | 10 ++++
.../org/apache/nifi/web/api/dto/DtoFactory.java | 9 ++++
.../WEB-INF/partials/canvas/queue-listing.jsp | 8 +--
.../src/main/webapp/css/queue-listing.css | 32 +++++-------
.../webapp/js/nf/canvas/nf-queue-listing.js | 54 ++++++++++++++++----
8 files changed, 99 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileDTO.java
index ec64e3f..95e91ef 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileDTO.java
@@ -18,8 +18,10 @@ package org.apache.nifi.web.api.dto;
import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
import java.util.Map;
+@XmlType(name = "flowFile")
public class FlowFileDTO extends FlowFileSummaryDTO {
private Map<String, String> attributes;
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileSummaryDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileSummaryDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileSummaryDTO.java
index e9bf9c5..2a8a1f1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileSummaryDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowFileSummaryDTO.java
@@ -18,6 +18,9 @@ package org.apache.nifi.web.api.dto;
import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
+
+@XmlType(name = "flowFileSummary")
public class FlowFileSummaryDTO {
private String uri;
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ListingRequestDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ListingRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ListingRequestDTO.java
index 36c0518..c058ae3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ListingRequestDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ListingRequestDTO.java
@@ -19,12 +19,14 @@ package org.apache.nifi.web.api.dto;
import java.util.Date;
import java.util.List;
+import javax.xml.bind.annotation.XmlType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.apache.nifi.web.api.dto.util.TimestampAdapter;
import com.wordnik.swagger.annotations.ApiModelProperty;
+@XmlType(name = "listingRequest")
public class ListingRequestDTO {
private String id;
@@ -43,6 +45,7 @@ public class ListingRequestDTO {
private Integer completedStepCount;
private String state;
+ private QueueSizeDTO queueSize;
private List<FlowFileSummaryDTO> flowFileSummaries;
@@ -234,4 +237,16 @@ public class ListingRequestDTO {
public void setCompletedStepCount(Integer completedStepCount) {
this.completedStepCount = completedStepCount;
}
+
+ /**
+ * @return the size for the queue
+ */
+ @ApiModelProperty(value = "The size of the queue")
+ public QueueSizeDTO getQueueSize() {
+ return queueSize;
+ }
+
+ public void setQueueSize(QueueSizeDTO queueSize) {
+ this.queueSize = queueSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
index f01b1a5..55e58ac 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
@@ -209,6 +209,7 @@ import org.apache.nifi.web.api.dto.ListingRequestDTO;
import org.apache.nifi.web.api.dto.NodeDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
+import org.apache.nifi.web.api.dto.QueueSizeDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO;
@@ -2867,6 +2868,8 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
ListFlowFileState state = null;
int numStepsCompleted = 0;
int numStepsTotal = 0;
+ int objectCount = 0;
+ long byteCount = 0;
boolean finished = true;
for (final Map.Entry<NodeIdentifier, ListingRequestDTO> entry : listingRequestMap.entrySet()) {
final NodeIdentifier nodeIdentifier = entry.getKey();
@@ -2877,6 +2880,10 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
numStepsCompleted += nodeRequest.getCompletedStepCount();
numStepsTotal += nodeRequest.getTotalStepCount();
+ final QueueSizeDTO nodeQueueSize = nodeRequest.getQueueSize();
+ objectCount += nodeQueueSize.getObjectCount();
+ byteCount += nodeQueueSize.getByteCount();
+
if (!nodeRequest.getFinished()) {
finished = false;
}
@@ -2914,6 +2921,9 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
final int percentCompleted = numStepsCompleted / numStepsTotal;
listingRequest.setPercentCompleted(percentCompleted);
listingRequest.setFinished(finished);
+
+ listingRequest.getQueueSize().setByteCount(byteCount);
+ listingRequest.getQueueSize().setObjectCount(objectCount);
}
/**
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 9d02cbc..22031bb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -354,6 +354,13 @@ public final class DtoFactory {
return ListFlowFileState.COMPLETE.equals(state) || ListFlowFileState.CANCELED.equals(state) || ListFlowFileState.FAILURE.equals(state);
}
+ private QueueSizeDTO createQueueSizeDTO(final QueueSize queueSize) {
+ final QueueSizeDTO dto = new QueueSizeDTO();
+ dto.setByteCount(queueSize.getByteCount());
+ dto.setObjectCount(queueSize.getObjectCount());
+ return dto;
+ }
+
/**
* Creates a ListingRequestDTO from the specified ListFlowFileStatus.
*
@@ -375,6 +382,8 @@ public final class DtoFactory {
dto.setCompletedStepCount(listingRequest.getCompletedStepCount());
dto.setPercentCompleted(listingRequest.getCompletionPercentage());
+ dto.setQueueSize(createQueueSizeDTO(listingRequest.getQueueSize()));
+
if (isListingRequestComplete(listingRequest.getState())) {
final List<FlowFileSummary> flowFileSummaries = listingRequest.getFlowFileSummaries();
if (flowFileSummaries != null) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/queue-listing.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/queue-listing.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/queue-listing.jsp
index d00e365..0d8435a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/queue-listing.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/queue-listing.jsp
@@ -16,12 +16,12 @@
--%>
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<div id="queue-listing-container" class="hidden">
- <div id="queue-listing-header-and-filter">
+ <div id="queue-listing-header-container">
<div id="queue-listing-header-text"></div>
</div>
- <div id="queue-listing-refresh-container">
- <div id="queue-listing-last-refreshed-container">
- Last updated: <span id="queue-listing-last-refreshed"></span>
+ <div id="queue-listing-stats-container">
+ <div id="queue-listing-stats">
+ Displaying <span id="displayed-flowfiles"></span> of <span id="total-flowfiles-count"></span> (<span id="total-flowfiles-size"></span>)
</div>
<div id="queue-listing-loading-container" class="loading-container"></div>
</div>
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css
index f705ceb..2d7205e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
/*
Queue listing styles
*/
@@ -26,7 +27,7 @@
right: 0px;
}
-#queue-listing-header-and-filter {
+#queue-listing-header-container {
height: 35px;
margin-top: 20px;
margin-left: 20px;
@@ -39,21 +40,20 @@
font-weight: bold;
}
-#queue-listing-refresh-container {
- height: 26px;
- margin-left: 20px;
- margin-right: 20px;
- margin-top: 18px;
+#queue-listing-stats-container {
+ margin-left: 15px;
+ margin-top: 30px;
-webkit-user-select: none;
-moz-user-select: none;
}
-#queue-listing-last-refreshed-container {
- float: left;
- color: #666;
- font-weight: normal;
- margin-top: 6px;
- margin-left: 3px;
+#queue-listing-stats {
+ font-size: 9px;
+ font-weight: bold;
+ color: #9f6000;
+ clear: left;
+ line-height: normal;
+ margin-left: 5px;
}
#queue-listing-loading-container {
@@ -65,14 +65,6 @@
margin-left: 3px;
}
-#queue-listing-last-refreshed {
- font-weight: bold;
-}
-
-#queue-listing-header {
- padding-top: 10px;
-}
-
/* queue listing table */
#queue-listing-table {
http://git-wip-us.apache.org/repos/asf/nifi/blob/ba54e8e3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-queue-listing.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-queue-listing.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-queue-listing.js
index 76bea54..cba598a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-queue-listing.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-queue-listing.js
@@ -180,6 +180,11 @@ nf.QueueListing = (function () {
// use the listing request from when the listing completed
if (nf.Common.isDefinedAndNotNull(listingRequest.flowFileSummaries)) {
+ // update the queue size
+ $('#total-flowfiles-count').text(nf.Common.formatInteger(listingRequest.queueSize.objectCount));
+ $('#total-flowfiles-size').text(nf.Common.formatDataSize(listingRequest.queueSize.byteCount));
+
+ // get the grid to load the data
var queueListingGrid = $('#queue-listing-table').data('gridInstance');
var queueListingData = queueListingGrid.getData();
@@ -195,7 +200,7 @@ nf.QueueListing = (function () {
});
}
}
- }).fail(nf.Common.handleAjaxError);
+ });
} else {
// close the dialog
$('#listing-request-status-dialog').modal('hide');
@@ -308,10 +313,21 @@ nf.QueueListing = (function () {
};
// function for formatting durations
- var formatDuration = function (row, cell, value, columnDef, dataContext) {
+ var durationFormatter = function (row, cell, value, columnDef, dataContext) {
return nf.Common.formatDuration(value);
}
+ // function for formatting penalization
+ var penalizedFormatter = function (row, cell, value, columnDef, dataContext) {
+ var markup = '';
+
+ if (value === true) {
+ markup += 'Yes';
+ }
+
+ return markup;
+ }
+
// initialize the queue listing table
var queueListingColumns = [
{id: 'moreDetails', field: 'moreDetails', name: ' ', sortable: false, resizable: false, formatter: moreDetailsFormatter, width: 50, maxWidth: 50},
@@ -319,9 +335,9 @@ nf.QueueListing = (function () {
{id: 'FLOWFILE_UUID', name: 'UUID', field: 'uuid', sortable: true, resizable: true},
{id: 'FILENAME', name: 'Filename', field: 'filename', sortable: true, resizable: true},
{id: 'FLOWFILE_SIZE', name: 'File Size', field: 'size', sortable: true, resizable: true, defaultSortAsc: false, formatter: dataSizeFormatter},
- {id: 'QUEUED_DURATION', name: 'Queued Duration', field: 'queuedDuration', sortable: true, resizable: true, formatter: formatDuration},
- {id: 'FLOWFILE_AGE', name: 'Lineage Duration', field: 'lineageDuration', sortable: true, resizable: true, formatter: formatDuration},
- {id: 'PENALIZATION', name: 'Penalized', field: 'penalized', sortable: true, resizable: false, width: 100, maxWidth: 100}
+ {id: 'QUEUED_DURATION', name: 'Queued Duration', field: 'queuedDuration', sortable: true, resizable: true, formatter: durationFormatter},
+ {id: 'FLOWFILE_AGE', name: 'Lineage Duration', field: 'lineageDuration', sortable: true, resizable: true, formatter: durationFormatter},
+ {id: 'PENALIZATION', name: 'Penalized', field: 'penalized', sortable: true, resizable: false, width: 100, maxWidth: 100, formatter: penalizedFormatter}
];
// conditionally show the cluster node identifier
@@ -365,10 +381,6 @@ nf.QueueListing = (function () {
if (target.hasClass('show-flowfile-details')) {
showFlowFileDetails(item);
}
- } else if (queueListingGrid.getColumns()[args.cell].id === 'actions') {
- if (target.hasClass('delete-flowfile')) {
- deleteFlowfile(item);
- }
}
});
@@ -377,7 +389,7 @@ nf.QueueListing = (function () {
queueListingGrid.updateRowCount();
queueListingGrid.render();
- // update the total number of displayed processors
+ // update the total number of displayed flowfiles
$('#displayed-flowfiles').text(args.current);
});
queueListingData.onRowsChanged.subscribe(function (e, args) {
@@ -398,9 +410,31 @@ nf.QueueListing = (function () {
* @param {object} The connection
*/
listQueue: function (connection) {
+ // update stats
+ $('#displayed-flowfiles, #total-flowfiles-count').text('0');
+ $('#total-flowfiles-size').text(nf.Common.formatDataSize(0));
+
+ // update the connection name
+ var connectionName = nf.CanvasUtils.formatConnectionName(connection.component);
+ if (connectionName === '') {
+ connectionName = 'Connection';
+ }
+ $('#queue-listing-header-text').text(connectionName);
+
// show the listing container
nf.Shell.showContent('#queue-listing-container').done(function () {
$('#queue-listing-table').removeData('connection');
+
+ // clear the table
+ var queueListingGrid = $('#queue-listing-table').data('gridInstance');
+ if (nf.Common.isDefinedAndNotNull(queueListingGrid)) {
+ var queueListingData = queueListingGrid.getData();
+
+ // clear the flowfiles
+ queueListingData.beginUpdate();
+ queueListingData.setItems([], 'uuid');
+ queueListingData.endUpdate();
+ }
});
// adjust the table size