You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/10/29 05:43:41 UTC

Change in asterixdb[master]: [ASTERIXDB-2150] Add feed log file path to the feed stats

Xikui Wang has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2109

Change subject: [ASTERIXDB-2150] Add feed log file path to the feed stats
......................................................................

[ASTERIXDB-2150] Add feed log file path to the feed stats

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

We add the feed log file path to the feed stats. The user could create
an external dataset based on the file path to access the feed logs for
future debugging.

Change-Id: I3225c671a4b540f13d3beb001ba503743be5e886
---
M asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
D asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.regexadm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
M asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/StartFeedStatement.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
13 files changed, 101 insertions(+), 45 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/09/2109/1

diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index 264e9bc..fad49db 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -87,6 +87,10 @@
         return runtimes.get(runtimeId);
     }
 
+    public String getNodeId() {
+        return this.nodeId;
+    }
+
     @Override
     public String toString() {
         return ActiveManager.class.getSimpleName() + "[" + nodeId + "]";
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index bae04d5..8534a96 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -51,6 +51,7 @@
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -262,13 +263,7 @@
     }
 
     public String formatStats(List<String> responses) {
-        StringBuilder strBuilder = new StringBuilder();
-        strBuilder.append("{\"Stats\": [").append(responses.get(0));
-        for (int i = 1; i < responses.size(); i++) {
-            strBuilder.append(", ").append(responses.get(i));
-        }
-        strBuilder.append("]}");
-        return strBuilder.toString();
+        return "[" + StringUtils.join(responses, ",") + "]";
     }
 
     @SuppressWarnings("unchecked")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm
deleted file mode 100644
index 0fcdf15..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-  "experiments.TweetFeed(Feed)" : {
-    "Stats" : [ {
-      "adapter-stats" : {
-        "incoming-records-count" : 13,
-        "failed-at-parser-records-count" : 3
-      }
-    } ]
-  }
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.regexadm
new file mode 100644
index 0000000..eb1dbdc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.regexadm
@@ -0,0 +1,9 @@
+\{
+  "experiments\.TweetFeed\(Feed\)" : \[ \{
+    "adapter-partition-id" : 0,
+    "node_id" : "asterix_nc\d",
+    "incoming-records-count" : 13,
+    "failed-at-parser-records-count" : 3,
+    "failed-record-log-file-location" : "[^"]+"
+  \} \]
+\}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
index 47560f6..00429c1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -297,6 +297,16 @@
         <expected-error>Function fundv.test_func0@1 is being used. It cannot be dropped</expected-error>
       </compilation-unit>
     </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="feed-with-log-dataset-metadata">
+        <output-dir compare="Text">feed-with-log-dataset-metadata</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="feed-with-log-dataset-runtime">
+        <output-dir compare="Text">feed-with-log-dataset-runtime</output-dir>
+      </compilation-unit>
+    </test-case>
   </test-group>
   <test-group name="upsert">
     <test-case FilePath="upsert">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 824f51a..cb0b0cf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -18,6 +18,7 @@
  */
 package org.apache.asterix.external.dataflow;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
@@ -36,8 +37,6 @@
 import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
 
 public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowController {
-    public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = "incoming-records-count";
-    public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = "failed-at-parser-records-count";
 
     public enum State {
         CREATED,
@@ -281,7 +280,10 @@
 
     @Override
     public String getStats() {
-        return "{\"" + INCOMING_RECORDS_COUNT_FIELD_NAME + "\": " + incomingRecordsCount + ", \"" +
-                FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME + "\": " + failedRecordsCount + "}";
+        return "\"" + ExternalDataConstants.INCOMING_RECORDS_COUNT_FIELD_NAME + "\": " + incomingRecordsCount + ", \""
+                + ExternalDataConstants.FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME + "\": " + failedRecordsCount + ", \""
+                + ExternalDataConstants.FAILED_RECORD_LOG_LOCATION_NAME + "\": \""
+                + feedLogManager.getDir().toAbsolutePath() + File.separator + FeedLogManager.BAD_RECORDS_FILE_NAME
+                + "\"";
     }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
index 7907e69..ae83bb5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
@@ -26,6 +26,7 @@
 import org.apache.asterix.active.EntityId;
 import org.apache.asterix.external.api.IAdapterFactory;
 import org.apache.asterix.external.dataset.adapter.FeedAdapter;
+import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.hyracks.api.comm.IFrame;
 import org.apache.hyracks.api.comm.VSizeFrame;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -143,10 +144,8 @@
 
     @Override
     public String getStats() {
-        if (adapter != null) {
-            return "{\"adapter-stats\": " + adapter.getStats() + "}";
-        } else {
-            return "\"Runtime stats is not available.\"";
-        }
+        return "{\"" + ExternalDataConstants.PARTITION_ID_NAME + "\": " + this.runtimeId.getPartition() + ", " + "\""
+                + ExternalDataConstants.FEED_NODE_ID_NAME + "\": \"" + this.activeManager.getNodeId() + "\", "
+                + adapter.getStats() + "}";
     }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index a6c46c3..bc336c5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -225,5 +225,15 @@
     public static final String READER_RSS = "rss_feed";
     public static final String FORMAT_CSV = "csv";
 
+
+    /**
+     * External Runtime stats attributes
+     * */
+    public static final String PARTITION_ID_NAME = "adapter-partition-id";
+    public static final String FEED_NODE_ID_NAME = "node_id";
+    public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = "incoming-records-count";
+    public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = "failed-at-parser-records-count";
+    public static final String FAILED_RECORD_LOG_LOCATION_NAME = "failed-record-log-file-location";
+
     public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
index 9d887b6..5e031b0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
@@ -32,6 +32,8 @@
 import java.util.Date;
 import java.util.TreeSet;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
@@ -50,6 +52,12 @@
     public static final String START_PREFIX = "s:";
     public static final String END_PREFIX = "e:";
     private static final String DATE_FORMAT_STRING = "MM/dd/yyyy HH:mm:ss";
+
+    public static final String FIELD_NAME_FAILED_RECORD_ID = "FailedRecordID";
+    public static final String FIELD_NAME_FAILED_RECORD_ERROR_MSG = "FailedRecordErrorMessage";
+    public static final String FIELD_NAME_FAILED_RECORD_TIME = "FailedTimestamp";
+    public static final String FIELD_NAME_FAILED_RECORD_RECORD = "FailedRecord";
+
     public static final int PREFIX_SIZE = START_PREFIX.length() + DATE_FORMAT_STRING.length() + 1;
     private String currentPartition;
     private final TreeSet<String> completed;
@@ -60,6 +68,8 @@
     private final StringBuilder stringBuilder = new StringBuilder();
     private int count = 0;
     private static final DateFormat df = new SimpleDateFormat(DATE_FORMAT_STRING);
+    private final ObjectMapper logRecordMapper = new ObjectMapper();
+    private final ObjectNode errorRecordNode = logRecordMapper.createObjectNode();
 
     public FeedLogManager(File file) throws HyracksDataException {
         try {
@@ -171,13 +181,13 @@
 
     public synchronized void logRecord(String record, String errorMessage) throws IOException {
         stringBuilder.setLength(0);
-        stringBuilder.append(record);
-        stringBuilder.append(ExternalDataConstants.LF);
-        stringBuilder.append(df.format((new Date())));
-        stringBuilder.append(' ');
-        stringBuilder.append(errorMessage);
-        stringBuilder.append(ExternalDataConstants.LF);
-        recordLogger.write(stringBuilder.toString());
+        Date logTime = new Date();
+        errorRecordNode.removeAll();
+        errorRecordNode.put(FIELD_NAME_FAILED_RECORD_ID, logTime.getTime());
+        errorRecordNode.put(FIELD_NAME_FAILED_RECORD_TIME, df.format(logTime));
+        errorRecordNode.put(FIELD_NAME_FAILED_RECORD_ERROR_MSG, errorMessage);
+        errorRecordNode.put(FIELD_NAME_FAILED_RECORD_RECORD, record);
+        recordLogger.write(errorRecordNode.toString() + ExternalDataConstants.LF);
         recordLogger.flush();
     }
 
@@ -188,4 +198,8 @@
     public synchronized boolean isSplitRead(String split) {
         return completed.contains(split);
     }
+
+    public Path getDir() {
+        return this.dir;
+    }
 }
diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 0cc6789..d18df81 100644
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -462,9 +462,7 @@
     ( <HINTS> hints = Properties() )?
     ( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
       {
-        ExternalDetailsDecl edd = new ExternalDetailsDecl();
-        edd.setAdapter(adapterName);
-        edd.setProperties(properties);
+        ExternalDetailsDecl edd = new ExternalDetailsDecl(adapterName, properties);
         dsetDecl = new DatasetDecl(nameComponents.first,
                                    nameComponents.second,
                                    typeComponents.first,
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java
index b15579c..bcc56eb 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java
@@ -21,14 +21,11 @@
 import java.util.Map;
 
 public class ExternalDetailsDecl implements IDatasetDetailsDecl {
-    private Map<String, String> properties;
-    private String adapter;
+    private final Map<String, String> properties;
+    private final String adapter;
 
-    public void setAdapter(String adapter) {
+    public ExternalDetailsDecl(String adapter, Map<String, String> properties) {
         this.adapter = adapter;
-    }
-
-    public void setProperties(Map<String, String> properties) {
         this.properties = properties;
     }
 
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/StartFeedStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/StartFeedStatement.java
index b3452b5..278bcc1 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/StartFeedStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/StartFeedStatement.java
@@ -19,11 +19,40 @@
 
 package org.apache.asterix.lang.common.statement;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.asterix.active.IActiveEntityEventsListener;
+import org.apache.asterix.common.api.IMetadataLockManager;
+import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.utils.JobUtils;
+import org.apache.asterix.external.feed.watch.StatsSubscriber;
+import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.lang.common.struct.Identifier;
 import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
+import org.apache.asterix.metadata.utils.DatasetUtil;
+import org.apache.asterix.metadata.utils.MetadataLockUtil;
+import org.apache.asterix.metadata.utils.MetadataUtil;
+import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableObject;
 import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 public class StartFeedStatement implements Statement {
 
@@ -58,4 +87,5 @@
     public Identifier getFeedName() {
         return feedName;
     }
+
 }
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index d3a64d8..5fef8e4 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -513,9 +513,7 @@
     ( <HINTS> hints = Properties() )?
     ( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy() (LOOKAHEAD(1) compactionPolicyProperties = Configuration())? )?
       {
-        ExternalDetailsDecl edd = new ExternalDetailsDecl();
-        edd.setAdapter(adapterName);
-        edd.setProperties(properties);
+        ExternalDetailsDecl edd = new ExternalDetailsDecl(adapterName, properties);
         dsetDecl = new DatasetDecl(nameComponents.first,
                                    nameComponents.second,
                                    typeComponents.first,

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2109
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3225c671a4b540f13d3beb001ba503743be5e886
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>