You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2015/04/29 18:21:09 UTC
[2/2] incubator-nifi git commit: NIFI-555: Create index name based
off of event time of first event in index, not based on creation time of index
NIFI-555: Create index name based off of event time of first event in index, not based on creation time of index
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/4baf48ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/4baf48ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/4baf48ae
Branch: refs/heads/develop
Commit: 4baf48ae958ea5f084f50b0b74d7c0efe417a29f
Parents: 20831c8
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Apr 29 09:41:40 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Apr 29 09:41:40 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/provenance/IndexConfiguration.java | 16 ++++++++--------
.../provenance/PersistentProvenanceRepository.java | 6 +++++-
.../apache/nifi/provenance/lucene/LineageQuery.java | 4 ++--
3 files changed, 15 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4baf48ae/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
index 3beab65..9ea793d 100644
--- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
+++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
@@ -121,13 +121,13 @@ public class IndexConfiguration {
}
}
- public File getWritableIndexDirectory(final File provenanceLogFile) {
+ public File getWritableIndexDirectory(final File provenanceLogFile, final long newIndexTimestamp) {
lock.lock();
try {
final File storageDirectory = provenanceLogFile.getParentFile();
List<File> indexDirectories = this.indexDirectoryMap.get(storageDirectory);
if (indexDirectories == null) {
- final File newDir = addNewIndex(storageDirectory, provenanceLogFile);
+ final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp);
indexDirectories = new ArrayList<>();
indexDirectories.add(newDir);
indexDirectoryMap.put(storageDirectory, indexDirectories);
@@ -135,7 +135,7 @@ public class IndexConfiguration {
}
if (indexDirectories.isEmpty()) {
- final File newDir = addNewIndex(storageDirectory, provenanceLogFile);
+ final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp);
indexDirectories.add(newDir);
return newDir;
}
@@ -143,7 +143,7 @@ public class IndexConfiguration {
final File lastDir = indexDirectories.get(indexDirectories.size() - 1);
final long size = getSize(lastDir);
if (size > repoConfig.getDesiredIndexSize()) {
- final File newDir = addNewIndex(storageDirectory, provenanceLogFile);
+ final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp);
indexDirectories.add(newDir);
return newDir;
} else {
@@ -154,14 +154,14 @@ public class IndexConfiguration {
}
}
- private File addNewIndex(final File storageDirectory, final File provenanceLogFile) {
+ private File addNewIndex(final File storageDirectory, final File provenanceLogFile, final long newIndexTimestamp) {
// Build the event time of the first record into the index's filename so that we can determine
// which index files to look at when we perform a search. We use the timestamp of the first record
// in the Provenance Log file, rather than the current time, because we may perform the Indexing
// retroactively.
Long firstEntryTime = getFirstEntryTime(provenanceLogFile);
if (firstEntryTime == null) {
- firstEntryTime = System.currentTimeMillis();
+ firstEntryTime = newIndexTimestamp;
}
return new File(storageDirectory, "index-" + firstEntryTime);
}
@@ -222,7 +222,7 @@ public class IndexConfiguration {
}
});
- for (File indexDir : sortedIndexDirectories) {
+ for (final File indexDir : sortedIndexDirectories) {
// If the index was last modified before the start time, we know that it doesn't
// contain any data for us to query.
if (startTime != null && indexDir.lastModified() < startTime) {
@@ -282,7 +282,7 @@ public class IndexConfiguration {
}
boolean foundIndexCreatedLater = false;
- for (File indexDir : sortedIndexDirectories) {
+ for (final File indexDir : sortedIndexDirectories) {
// If the index was last modified before the log file was created, we know the index doesn't include
// any data for the provenance log.
if (indexDir.lastModified() < firstEntryTime) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4baf48ae/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
index 214fc7c..3bdd38f 100644
--- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
+++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
@@ -1231,6 +1231,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
}
});
+ long earliestTimestamp = System.currentTimeMillis();
for (final RecordReader reader : readers) {
StandardProvenanceEventRecord record = null;
@@ -1252,6 +1253,9 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
continue;
}
+ if ( record.getEventTime() < earliestTimestamp ) {
+ earliestTimestamp = record.getEventTime();
+ }
recordToReaderMap.put(record, reader);
}
@@ -1262,7 +1266,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
final IndexingAction indexingAction = new IndexingAction(this);
- final File indexingDirectory = indexConfig.getWritableIndexDirectory(writerFile);
+ final File indexingDirectory = indexConfig.getWritableIndexDirectory(writerFile, earliestTimestamp);
final IndexWriter indexWriter = indexManager.borrowIndexWriter(indexingDirectory);
try {
long maxId = 0L;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4baf48ae/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
index 5e4f69d..502068b 100644
--- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
+++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
@@ -96,8 +96,8 @@ public class LineageQuery {
final DocsReader docsReader = new DocsReader(repo.getConfiguration().getStorageDirectories());
final Set<ProvenanceEventRecord> recs = docsReader.read(uuidQueryTopDocs, searcher.getIndexReader(), repo.getAllLogFiles(), new AtomicInteger(0), Integer.MAX_VALUE);
final long readDocsEnd = System.nanoTime();
- logger.debug("Finished Lineage Query; Lucene search took {} millis, reading records took {} millis",
- TimeUnit.NANOSECONDS.toMillis(searchEnd - searchStart), TimeUnit.NANOSECONDS.toMillis(readDocsEnd - searchEnd));
+ logger.debug("Finished Lineage Query against {}; Lucene search took {} millis, reading records took {} millis",
+ indexDirectory, TimeUnit.NANOSECONDS.toMillis(searchEnd - searchStart), TimeUnit.NANOSECONDS.toMillis(readDocsEnd - searchEnd));
return recs;
} finally {