You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2021/03/26 11:11:22 UTC
[asterixdb] 05/15: [NO ISSUE][*DB][EXT]
FeedRecordDataFlowController stats += timestamp
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 706b2e9fc8336679a70499d028ed1f43ffedca25
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed Mar 17 00:01:06 2021 -0400
[NO ISSUE][*DB][EXT] FeedRecordDataFlowController stats += timestamp
also, += ThrowingConsumer.forEach() utility method
Change-Id: I53ed2f4a9637de00b90a4d7c637653d440cd3740
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10564
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../results/api/feed-stats/feed-stats.1.adm | 10 ---
.../results/api/feed-stats/feed-stats.5.regexjson | 13 ++++
.../dataflow/FeedRecordDataFlowController.java | 10 ++-
.../lang/common/util/DataverseNameUtils.java | 85 ----------------------
.../org/apache/hyracks/util/ThrowingConsumer.java | 14 ++++
5 files changed, 33 insertions(+), 99 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm
deleted file mode 100644
index 0fcdf15..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "experiments.TweetFeed(Feed)" : {
- "Stats" : [ {
- "adapter-stats" : {
- "incoming-records-count" : 13,
- "failed-at-parser-records-count" : 3
- }
- } ]
- }
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson
new file mode 100644
index 0000000..54028c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson
@@ -0,0 +1,13 @@
+{
+ "experiments.TweetFeed(Feed)": {
+ "Stats": [
+ {
+ "adapter-stats": {
+ "timestamp": "R{[0-9]+}",
+ "incoming-records-count": 13,
+ "failed-at-parser-records-count": 3
+ }
+ }
+ ]
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 3fb7db7..56257a8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -42,6 +42,7 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = "incoming-records-count";
public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = "failed-at-parser-records-count";
public static final String READER_STATS_FIELD_NAME = "reader-stats";
+ public static final String TIMESTAMP_FIELD_NAME = "timestamp";
public enum State {
CREATED,
@@ -271,11 +272,12 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
StringBuilder str = new StringBuilder();
str.append("{");
if (readerStats != null) {
- str.append("\"").append(READER_STATS_FIELD_NAME).append("\":").append(readerStats).append(", ");
+ str.append("\"" + READER_STATS_FIELD_NAME + "\":").append(readerStats).append(",");
}
- str.append("\"").append(INCOMING_RECORDS_COUNT_FIELD_NAME).append("\": ").append(incomingRecordsCount)
- .append(", \"").append(FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME).append("\": ")
- .append(failedRecordsCount).append("}");
+ str.append("\"" + TIMESTAMP_FIELD_NAME + "\":").append(System.currentTimeMillis()).append(",");
+ str.append("\"" + INCOMING_RECORDS_COUNT_FIELD_NAME + "\":").append(incomingRecordsCount)
+ .append(",\"" + FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME + "\":").append(failedRecordsCount)
+ .append("}");
return str.toString();
}
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java
deleted file mode 100644
index 1a3f26a..0000000
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.lang.common.util;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.lang.common.visitor.FormatPrintVisitor;
-
-public class DataverseNameUtils {
-
- static protected Set<Character> validIdentifierChars = new HashSet<>();
- static protected Set<Character> validIdentifierStartChars = new HashSet<>();
-
- static {
- for (char ch = 'a'; ch <= 'z'; ++ch) {
- validIdentifierChars.add(ch);
- validIdentifierStartChars.add(ch);
- }
- for (char ch = 'A'; ch <= 'Z'; ++ch) {
- validIdentifierChars.add(ch);
- validIdentifierStartChars.add(ch);
- }
- for (char ch = '0'; ch <= '9'; ++ch) {
- validIdentifierChars.add(ch);
- }
- validIdentifierChars.add('_');
- validIdentifierChars.add('$');
- }
-
- protected static boolean needQuotes(String str) {
- if (str.length() == 0) {
- return false;
- }
- if (!validIdentifierStartChars.contains(str.charAt(0))) {
- return true;
- }
- for (char ch : str.toCharArray()) {
- if (!validIdentifierChars.contains(ch)) {
- return true;
- }
- }
- return false;
- }
-
- protected static String normalize(String str) {
- if (needQuotes(str)) {
- return FormatPrintVisitor.revertStringToQuoted(str);
- }
- return str;
- }
-
- public static String generateDataverseName(DataverseName dataverseName) {
- List<String> dataverseNameParts = new ArrayList<>();
- StringBuilder sb = new StringBuilder();
- dataverseNameParts.clear();
- dataverseName.getParts(dataverseNameParts);
- for (int i = 0, ln = dataverseNameParts.size(); i < ln; i++) {
- if (i > 0) {
- sb.append(DataverseName.CANONICAL_FORM_SEPARATOR_CHAR);
- }
- sb.append(normalize(dataverseNameParts.get(i)));
- }
- return sb.toString();
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
index 4f73680..36b5cf0 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
@@ -46,4 +46,18 @@ public interface ThrowingConsumer<V> {
return null;
};
}
+
+ static <R> void forEach(Iterable<R> iterable, ThrowingConsumer<R> consumer) throws Exception {
+ try {
+ iterable.forEach(value -> {
+ try {
+ consumer.process(value);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ });
+ } catch (UncheckedExecutionException e) {
+ throw (Exception) e.getCause();
+ }
+ }
}