You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by oz...@apache.org on 2017/03/03 14:19:14 UTC
nifi git commit: NIFI-3523: Ensure that if we are unable to read
Provenance events on startup that we don't prevent NiFi from starting
Repository: nifi
Updated Branches:
refs/heads/master 4ed64e756 -> 6c44d6be2
NIFI-3523: Ensure that if we are unable to read Provenance events on startup that we don't prevent NiFi from starting
This closes #1542
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6c44d6be
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6c44d6be
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6c44d6be
Branch: refs/heads/master
Commit: 6c44d6be2e31eae491f236f3be39eb3309c45cda
Parents: 4ed64e7
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Feb 24 11:45:26 2017 -0500
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Fri Mar 3 09:18:29 2017 -0500
----------------------------------------------------------------------
.../WriteAheadProvenanceRepository.java | 7 ++++++-
.../provenance/lucene/SimpleIndexManager.java | 2 ++
.../serialization/CompressableRecordReader.java | 15 ++++++++++++++-
.../store/WriteAheadStorePartition.java | 4 ++--
.../TestPersistentProvenanceRepository.java | 19 ++-----------------
5 files changed, 26 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/6c44d6be/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
index 229a96d..8975028 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
@@ -141,7 +141,12 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
eventStore.initialize();
eventIndex.initialize(eventStore);
- eventStore.reindexLatestEvents(eventIndex);
+ try {
+ eventStore.reindexLatestEvents(eventIndex);
+ } catch (final Exception e) {
+ logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
+ + "events will not be available from the Provenance Repository when a query is issued.", e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/6c44d6be/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
index b0b01e5..4d6c11d 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
@@ -318,7 +318,9 @@ public class SimpleIndexManager implements IndexManager {
// This method exists solely for unit testing purposes.
protected void close(final IndexWriterCount count) throws IOException {
+ logger.debug("Closing Index Writer for {}...", count.getWriter().getDirectory());
count.close();
+ logger.debug("Finished closing Index Writer for {}...", count.getWriter().getDirectory());
}
protected int getWriterCount() {
http://git-wip-us.apache.org/repos/asf/nifi/blob/6c44d6be/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
index 1a6c3c5..93c0669 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
@@ -283,7 +283,20 @@ public abstract class CompressableRecordReader implements RecordReader {
}
if (isData()) {
- return nextRecord(dis, serializationVersion);
+ while (true) {
+ try {
+ return nextRecord(dis, serializationVersion);
+ } catch (final IOException ioe) {
+ throw ioe;
+ } catch (final Exception e) {
+ // This would only happen if a bug were to exist such that an 'invalid' event were written
+ // out. For example an Event that has no FlowFile UUID. While there is in fact an underlying
+ // cause that would need to be sorted out in this case, the Provenance Repository should be
+ // resilient enough to handle this. Otherwise, we end up throwing an Exception, which may
+ // prevent iterating over additional events in the repository.
+ logger.error("Failed to read Provenance Event from " + filename + "; will skip this event and continue reading subsequent events", e);
+ }
+ }
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6c44d6be/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
index a25043a..fde76f5 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
@@ -128,8 +128,8 @@ public class WriteAheadStorePartition implements EventStorePartition {
maxEventId = eventId;
break;
}
- } catch (final IOException ioe) {
- logger.warn("Could not read file {}; if this file contains Provenance Events, new events may be created with the same event identifiers", file, ioe);
+ } catch (final Exception e) {
+ logger.warn("Could not read file {}; if this file contains Provenance Events, new events may be created with the same event identifiers", file, e);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6c44d6be/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
index 48d8e09..e3f729b 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
@@ -1905,13 +1905,6 @@ public class TestPersistentProvenanceRepository {
testRepo.recoverJournalFiles();
- assertEquals("mergeJournals() should report a skipped journal", 1, reportedEvents.size());
- assertEquals("mergeJournals() should report a skipped journal",
- "Failed to read Provenance Event Record from Journal due to java.lang.IllegalArgumentException: "
- + "No enum constant org.apache.nifi.provenance.ProvenanceEventType.BADTYPE; it's possible "
- + "that the record wasn't completely written to the file. This journal will be skipped.",
- reportedEvents.get(reportedEvents.size() - 1).getMessage());
-
final File storageDir = config.getStorageDirectories().values().iterator().next();
assertTrue(checkJournalRecords(storageDir, false) < 10000);
}
@@ -1937,7 +1930,7 @@ public class TestPersistentProvenanceRepository {
final ProvenanceEventRecord record = builder.build();
final ExecutorService exec = Executors.newFixedThreadPool(10);
- final List<Future> futures = new ArrayList<>();
+ final List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
futures.add(exec.submit(new Runnable() {
@Override
@@ -1948,7 +1941,7 @@ public class TestPersistentProvenanceRepository {
}
// corrupt the first record of the first journal file
- for (Future future : futures) {
+ for (Future<?> future : futures) {
while (!future.isDone()) {
Thread.sleep(10);
}
@@ -1958,14 +1951,6 @@ public class TestPersistentProvenanceRepository {
testRepo.recoverJournalFiles();
- assertEquals("mergeJournals should report a skipped journal", 1, reportedEvents.size());
- assertEquals("mergeJournals should report a skipped journal",
- "Failed to read Provenance Event Record from Journal due to java.lang.IllegalArgumentException: "
- + "No enum constant org.apache.nifi.provenance.ProvenanceEventType.BADTYPE; it's possible "
- + "that the record wasn't completely written to the file. The remainder of this journal will "
- + "be skipped.",
- reportedEvents.get(reportedEvents.size() - 1).getMessage());
-
final File storageDir = config.getStorageDirectories().values().iterator().next();
assertTrue(checkJournalRecords(storageDir, false) < 10000);
}