You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org> on 2016/03/15 16:36:59 UTC
Change in asterixdb[master]: Improve Error Handling in Feeds
abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/720
Change subject: Improve Error Handling in Feeds
......................................................................
Improve Error Handling in Feeds
This change separate parse exceptions from IOExceptions and
handle each differently. The responsibility of handling
IOExceptions falls on the datasource reader while records which
that couldn't be parsed end up added to the records log file.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitignore
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/couchbase/CouchbaseReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
20 files changed, 148 insertions(+), 31 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/1
diff --git a/.gitignore b/.gitignore
index f8e0b2d..cd497b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@
asterix-app/parserts/
asterix-app/opt_parserts/
asterix-app/runtime_parserts/
+asterix-app/data/csv/beer.csv
asterix-installer/ittest/
asterix-installer/repliationtest/
build
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index 769db19..fe3942d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -72,4 +72,9 @@
* progress and errors
*/
public void setFeedLogManager(FeedLogManager feedLogManager);
+
+ /**
+ * gives the record reader a chance to recover from IO errors during feed intake
+ */
+ public boolean handleException(Throwable th);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 3408af9..b1f219c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -63,11 +63,16 @@
continue;
}
tb.reset();
- dataParser.parse(record, tb.getDataOutput());
+ try {
+ dataParser.parse(record, tb.getDataOutput());
+ } catch (Exception e) {
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_PARSE_RECORD);
+ continue;
+ }
tb.addFieldEndOffset();
if (tb.getSize() > tupleForwarder.getMaxRecordSize()) {
// log
- feedLogManager.logRecord(record.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
continue;
}
tupleForwarder.addTuple(tb);
@@ -144,6 +149,7 @@
@Override
public boolean handleException(Throwable th) {
- return true;
+ // This is not a parser record. most likely, this error happened in the record reader.
+ return recordReader.handleException(th);
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
index 926022c..c9b7ee2 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
@@ -82,7 +82,7 @@
public void addTuple(ArrayTupleBuilder tb) throws HyracksDataException {
if (tb.getSize() > maxRecordSize) {
try {
- feedLogManager.logRecord(tb.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(tb.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
} catch (IOException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/couchbase/CouchbaseReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/couchbase/CouchbaseReader.java
index fe59aad..32411f5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/couchbase/CouchbaseReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/couchbase/CouchbaseReader.java
@@ -262,4 +262,9 @@
@Override
public void setFeedLogManager(FeedLogManager feedLogManager) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
index b162a02..f4ebff2 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
@@ -124,8 +124,9 @@
FileStatus fileStatus = hdfs.getFileStatus(new Path(fileName));
// Skip if not the same file stored in the files snapshot
if (fileStatus.getModificationTime() != snapshot.get(currentSplitIndex).getLastModefiedTime()
- .getTime())
+ .getTime()) {
continue;
+ }
}
reader.close();
@@ -192,4 +193,9 @@
@Override
public void setController(IDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
index 1af8695..c03e9e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
@@ -145,6 +145,11 @@
@Override
public void setController(IDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
class FetcherEventListenerImpl implements FetcherListener {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
index 2d6d8ea..f60e60d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
@@ -97,4 +97,9 @@
this.feedLogManager = feedLogManager;
reader.setFeedLogManager(feedLogManager);
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return reader.handleException(th);
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
index 617bc39..f0e9594 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
@@ -107,4 +107,9 @@
@Override
public void setController(IDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
index 19f156c..0397150 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -133,4 +133,9 @@
@Override
public void setController(IDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
index 469e866..3715bde 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
@@ -37,4 +37,6 @@
// TODO: Find a better way to send notifications
public abstract void setFeedLogManager(FeedLogManager logManager);
+
+ public abstract boolean handleException(Throwable th);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
index 89008aa..644d3e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
@@ -92,4 +92,8 @@
System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position());
return charBuffer.position();
}
+
+ public boolean handleException(Throwable th) {
+ return in.handleException(th);
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
index 5b654eb..33e2044 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
@@ -99,4 +99,9 @@
@Override
public void setController(AbstractFeedDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
index 8dcd5b6..8785fdd 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
@@ -28,8 +28,10 @@
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.asterix.external.util.FileSystemWatcher;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.log4j.Logger;
public class LocalFileSystemInputStream extends AInputStream {
+ private static final Logger LOGGER = Logger.getLogger(LocalFileSystemInputStream.class.getName());
private final FileSystemWatcher watcher;
private FileInputStream in;
private byte lastByte;
@@ -138,4 +140,18 @@
public void configure(Map<String, String> configuration) throws IOException {
watcher.init();
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ if (th instanceof IOException) {
+ try {
+ watcher.init();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to initialize watcher during failure recovery", e);
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketInputStream.java
index 67c4493..579b19b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketInputStream.java
@@ -172,4 +172,9 @@
public void setController(AbstractFeedDataFlowController controller) {
this.controller = controller;
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
index bf9653d..64a1d8f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
@@ -129,6 +129,11 @@
@Override
public void setController(AbstractFeedDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return HDFSInputStreamProvider.this.handleException(th);
+ }
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
index cd4a3c1..bdb4ce3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
@@ -121,6 +121,11 @@
@Override
public void setController(AbstractFeedDataFlowController controller) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
private static class DataProvider implements Runnable {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 4b2826c..41f1015 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -205,5 +205,6 @@
* Expected parameter values
*/
public static final String PARAMETER_OF_SIZE_ONE = "Value of size 1";
- public static final String LARGE_RECORD_ERROR_MESSAGE = "Record is too large";
+ public static final String ERROR_LARGE_RECORD = "Record is too large";
+ public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
index 2e3b8ec..b40a0c4 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
@@ -35,14 +35,13 @@
import java.util.LinkedList;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
public class FileSystemWatcher {
private static Logger LOGGER = Logger.getLogger(FileSystemWatcher.class.getName());
- private final WatchService watcher;
+ private WatchService watcher;
private final HashMap<WatchKey, Path> keys;
private final LinkedList<File> files = new LinkedList<File>();
private Iterator<File> it;
@@ -55,7 +54,6 @@
private AbstractFeedDataFlowController controller;
public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) throws IOException {
- this.watcher = isFeed ? FileSystems.getDefault().newWatchService() : null;
this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
this.expression = expression;
this.path = inputResource;
@@ -66,12 +64,21 @@
this.logManager = feedLogManager;
}
- public void init() throws IOException {
+ public synchronized void init() throws IOException {
LinkedList<Path> dirs = null;
dirs = new LinkedList<Path>();
LocalFileSystemUtils.traverse(files, path.toFile(), expression, dirs);
it = files.iterator();
if (isFeed) {
+ keys.clear();
+ if (watcher != null) {
+ try {
+ watcher.close();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close watcher service", e);
+ }
+ }
+ watcher = FileSystems.getDefault().newWatchService();
for (Path path : dirs) {
register(path);
}
@@ -117,7 +124,7 @@
return (WatchEvent<T>) event;
}
- private void handleEvents(WatchKey key) {
+ private void handleEvents(WatchKey key) throws IOException {
// get dir associated with the key
Path dir = keys.get(key);
if (dir == null) {
@@ -135,6 +142,9 @@
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Overflow event. Some events might have been missed");
}
+ // need to read and validate all files.
+ //TODO: use btrees for all logs
+ init();
continue;
}
@@ -164,6 +174,7 @@
if (!done) {
if (watcher != null) {
watcher.close();
+ watcher = null;
}
if (logManager != null) {
if (current != null) {
@@ -197,7 +208,7 @@
return false;
}
- public boolean hasNext() throws HyracksDataException {
+ public boolean hasNext() throws IOException {
if (it.hasNext()) {
return true;
}
@@ -214,6 +225,7 @@
while (key != null) {
handleEvents(key);
if (endOfEvents(key)) {
+ close();
return false;
}
key = watcher.poll();
@@ -229,12 +241,18 @@
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Feed Closed");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
} catch (ClosedWatchServiceException e) {
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("The watcher has exited");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
}
handleEvents(key);
if (endOfEvents(key)) {
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
index 2882083..44a8101 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
@@ -847,8 +847,9 @@
private boolean parseExpression(ExprTreeHolder tree, boolean full) throws IOException {
TokenType tt;
- if (!parseLogicalORExpression(tree))
+ if (!parseLogicalORExpression(tree)) {
return false;
+ }
if ((tt = lexer.peekToken()) == TokenType.LEX_QMARK) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
@@ -880,8 +881,9 @@
// | LogicalORExpression '||' LogicalANDExpression
private boolean parseLogicalORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseLogicalANDExpression(tree))
+ if (!parseLogicalANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_OR) {
ExprTreeHolder treeL = tree;
ExprTreeHolder treeR = mutableExprPool.get();
@@ -902,8 +904,9 @@
// LogicalANDExpression .= InclusiveORExpression
// | LogicalANDExpression '&&' InclusiveORExpression
private boolean parseLogicalANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseInclusiveORExpression(tree))
+ if (!parseInclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_AND) {
ExprTreeHolder treeL = tree;
ExprTreeHolder treeR = mutableExprPool.get();
@@ -924,8 +927,9 @@
// InclusiveORExpression .= ExclusiveORExpression
// | InclusiveORExpression '|' ExclusiveORExpression
public boolean parseInclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseExclusiveORExpression(tree))
+ if (!parseExclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_OR) {
ExprTreeHolder treeL = tree;
ExprTreeHolder treeR = mutableExprPool.get();
@@ -946,8 +950,9 @@
// ExclusiveORExpression .= ANDExpression
// | ExclusiveORExpression '^' ANDExpression
private boolean parseExclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseANDExpression(tree))
+ if (!parseANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_XOR) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
@@ -968,8 +973,9 @@
// ANDExpression .= EqualityExpression
// | ANDExpression '&' EqualityExpression
private boolean parseANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseEqualityExpression(tree))
+ if (!parseEqualityExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_AND) {
ExprTreeHolder treeL = tree;
ExprTreeHolder treeR = mutableExprPool.get();
@@ -995,8 +1001,9 @@
private boolean parseEqualityExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
int op = Operation.OpKind_NO_OP;
- if (!parseRelationalExpression(tree))
+ if (!parseRelationalExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_EQUAL || tt == TokenType.LEX_NOT_EQUAL || tt == TokenType.LEX_META_EQUAL
|| tt == TokenType.LEX_META_NOT_EQUAL) {
@@ -1040,8 +1047,9 @@
// | RelationalExpression '>=' ShiftExpression
private boolean parseRelationalExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parseShiftExpression(tree))
+ if (!parseShiftExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_LESS_THAN || tt == TokenType.LEX_GREATER_THAN || tt == TokenType.LEX_LESS_OR_EQUAL
|| tt == TokenType.LEX_GREATER_OR_EQUAL) {
@@ -1084,8 +1092,9 @@
// | ShiftExpression '>>' AdditiveExpression
// | ShiftExpression '>>>' AditiveExpression
private boolean parseShiftExpression(ExprTreeHolder tree) throws IOException {
- if (!parseAdditiveExpression(tree))
+ if (!parseAdditiveExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_LEFT_SHIFT || tt == TokenType.LEX_RIGHT_SHIFT || tt == TokenType.LEX_URIGHT_SHIFT) {
@@ -1126,8 +1135,9 @@
// | AdditiveExpression '+' MultiplicativeExpression
// | AdditiveExpression '-' MultiplicativeExpression
private boolean parseAdditiveExpression(ExprTreeHolder tree) throws IOException {
- if (!parseMultiplicativeExpression(tree))
+ if (!parseMultiplicativeExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_PLUS || tt == TokenType.LEX_MINUS) {
@@ -1155,8 +1165,9 @@
// | MultiplicativeExpression '/' UnaryExpression
// | MultiplicativeExpression '%' UnaryExpression
private boolean parseMultiplicativeExpression(ExprTreeHolder tree) throws IOException {
- if (!parseUnaryExpression(tree))
+ if (!parseUnaryExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_MULTIPLY || tt == TokenType.LEX_DIVIDE || tt == TokenType.LEX_MODULUS) {
@@ -1238,8 +1249,9 @@
// | PostfixExpression '[' Expression ']'
private boolean parsePostfixExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parsePrimaryExpression(tree))
+ if (!parsePrimaryExpression(tree)) {
return false;
+ }
while ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_BOX || tt == TokenType.LEX_SELECTION) {
ExprTreeHolder treeL = tree;
ExprTreeHolder treeR = mutableExprPool.get();
@@ -1460,9 +1472,9 @@
// the next token must be a ',' or a ')'
// or it can be a ';' if using old ClassAd semantics
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false))
+ if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false)) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_PAREN) {
+ } else if (tt != TokenType.LEX_CLOSE_PAREN) {
argList.clear();
throw new HyracksDataException(
"expected LEX_COMMA or LEX_CLOSE_PAREN but got " + String.valueOf(Lexer.strLexToken(tt)));
@@ -1488,8 +1500,9 @@
public boolean parseClassAd(ClassAd ad, boolean full) throws IOException {
TokenType tt;
ad.clear();
- if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX)
+ if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX) {
return false;
+ }
tt = lexer.peekToken();
TokenValue tv = tokenValuePool.get();
ExprTreeHolder tree = mutableExprPool.get();
@@ -1587,9 +1600,9 @@
// the next token must be a ',' or a '}'
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA)
+ if (tt == TokenType.LEX_COMMA) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_BRACE) {
+ } else if (tt != TokenType.LEX_CLOSE_BRACE) {
throw new HyracksDataException("while parsing expression list: expected "
+ "LEX_CLOSE_BRACE or LEX_COMMA but got " + Lexer.strLexToken(tt));
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 12:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1035/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 12
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Ian Maxon (Code Review)" <do...@asterixdb.incubator.apache.org>.
Ian Maxon has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 10: Code-Review+1
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 10
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 6:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1003/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 5:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1002/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Feeds
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/986/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Feeds
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/970/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 6: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1003/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 10:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1007/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 10
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 4:
(15 comments)
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
File asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql:
Line 22: The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed.
> line break
Done
Line 23: The feed lasts a configured duration with data arriving at a configured rate (tweets per second).
> line break
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java:
Line 33: public abstract boolean handle(Throwable th);
> rename to handleException
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java:
Line 29:
> add java docs to methods
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java:
Line 70: public boolean handle(Throwable th);
> rename to handleException
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
File asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java:
Line 55
> return this TODO
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java:
Line 100: @Override
> These methods are usable only in HDFS case. Think about restructuring the i
I have a design that would solve this but would rather have it in a separate change!
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java:
Line 103: reader.reset();
> Make sure the reader is closed in case of exceptions
It will always be closed in a subsequent call to close()
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java:
Line 94: @Override
> remove the super class methods below
can't do that. super is an interface and I want to force every class to provide its implementation.
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java:
Line 129: @Override
> remove the super class methods below
can't do that. super is an interface and I want to force every class to provide its implementation.
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java:
Line 117: public void reset() {
> Missing implementation.
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java:
Line 41: throw new IOException("method not supported. use read(byte[] buffer, int offset, int length) instead");
> change to unsupported or use in.read
Done
Line 53: in.close();
> You don't need to close the inputStream as well as the socket. Just close o
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java:
Line 78: started = true;
> this will be set in start()
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
File asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java:
Line 58: public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) throws IOException {
> No need to throw
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has submitted this change and it was merged.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/720
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
---
M .gitattributes
M .gitignore
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
M asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
M asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
M asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
M asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
A asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
R asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
M asterix-external-data/src/test/resources/functional_tests.txt
M asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
M asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
M asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
136 files changed, 3,333 insertions(+), 2,851 deletions(-)
Approvals:
Murtadha Hubail: Looks good to me, approved
Jenkins: Verified
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 15
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Ian Maxon, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#12).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
M asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
M asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
M asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
M asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
A asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
R asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
M asterix-external-data/src/test/resources/functional_tests.txt
M asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
M asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
M asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
137 files changed, 3,331 insertions(+), 2,851 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/12
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 12
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 3:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/987/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#4).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
56 files changed, 872 insertions(+), 791 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/4
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 9:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1006/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 8:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1005/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Ian Maxon, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#13).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
M asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
M asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
M asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
M asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
A asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
R asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
M asterix-external-data/src/test/resources/functional_tests.txt
M asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
M asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
M asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
137 files changed, 3,331 insertions(+), 2,851 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/13
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 8: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1005/ : ABORTED
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 7: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1004/ : ABORTED
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 12: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1035/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 12
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 10: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1007/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 10
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Feeds
......................................................................
Patch Set 2: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/986/ : FAILURE
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#6).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 918 insertions(+), 803 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/6
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 5: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1002/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 13: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1036/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 11:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1034/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 11
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#3).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
56 files changed, 874 insertions(+), 793 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 4: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/988/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#5).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 902 insertions(+), 801 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/5
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 7:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1004/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 3: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/987/ : FAILURE
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 4:
(15 comments)
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
File asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql:
Line 22: The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed.
line break
Line 23: The feed lasts a configured duration with data arriving at a configured rate (tweets per second).
line break
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java:
Line 33: public abstract boolean handle(Throwable th);
rename to handleException
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java:
Line 29:
add java docs to methods
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java:
Line 70: public boolean handle(Throwable th);
rename to handleException
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
File asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java:
Line 55
return this TODO
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java:
Line 100: @Override
These methods are usable only in HDFS case. Think about restructuring the interfaces to avoid adding them here.
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java:
Line 103: reader.reset();
Make sure the reader is closed in case of exceptions
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java:
Line 94: @Override
remove the super class methods below
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java:
Line 129: @Override
remove the super class methods below
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java:
Line 117: public void reset() {
Missing implementation.
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java:
Line 41: throw new IOException("method not supported. use read(byte[] buffer, int offset, int length) instead");
change to unsupported or use in.read
Line 53: in.close();
You don't need to close the inputStream as well as the socket. Just close one of them.
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java:
Line 78: started = true;
this will be set in start()
https://asterix-gerrit.ics.uci.edu/#/c/720/4/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
File asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java:
Line 58: public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) throws IOException {
No need to throw
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 13:
(4 comments)
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
File asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java:
Line 169: frameWriter.nextFrame(frame);
revert
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java:
Line 94:
add TODO to restructure HDFS specific stuff.
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java:
Line 154: if (th.getCause().getMessage().contains("Malformed input stream")) {
change this to exception type
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
File asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java:
Line 56: private LinkedList<Path> dirs;
final
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 13:
comments addressed and rebased on master
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 4:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/988/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#2).
Change subject: Improve Error Handling in Feeds
......................................................................
Improve Error Handling in Feeds
This change separate parse exceptions from IOExceptions and
handle each differently. The responsibility of handling
IOExceptions falls on the datasource reader while records which
that couldn't be parsed end up added to the records log file.
In addition, some refactoring was performed on external data
to remove the stream provider layer of it. This will make
it consistent with the record reader.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
55 files changed, 845 insertions(+), 793 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#8).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 924 insertions(+), 802 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/8
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#9).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 930 insertions(+), 803 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/9
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Ian Maxon, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#14).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
M asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
M asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
M asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
M asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
A asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
R asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
M asterix-external-data/src/test/resources/functional_tests.txt
M asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
M asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
M asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
136 files changed, 3,333 insertions(+), 2,851 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/14
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 14
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Ian Maxon, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#11).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
M asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
M asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
M asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
M asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
M asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
M asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
A asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
R asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
C asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
M asterix-external-data/src/test/resources/functional_tests.txt
M asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
M asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
136 files changed, 3,360 insertions(+), 2,874 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/11
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 11
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#10).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 935 insertions(+), 805 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/10
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 10
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#7).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 924 insertions(+), 802 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/720/7
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 13:
(2 comments)
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java
File asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameCache.java:
Line 169: frameWriter.nextFrame(frame);
> revert
Done
https://asterix-gerrit.ics.uci.edu/#/c/720/13/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
File asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java:
Line 154: if (th.getCause().getMessage().contains("Malformed input stream")) {
> change this to exception type
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 14: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 14
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Feeds
......................................................................
Patch Set 1: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/970/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 11: Verified-1
Build Unstable
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1034/ : UNSTABLE
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 11
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 13:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1036/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 9: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1006/ : ABORTED
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 14:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1038/
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 14
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Improve Error Handling in Local Directory Feeds
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Patch Set 14: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1038/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 14
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>
Gerrit-Reviewer: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No