You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mj...@apache.org on 2017/06/20 20:42:42 UTC
[3/4] accumulo git commit: ACCUMULO-4657 Update MetadataTableUtil.java
ACCUMULO-4657 Update MetadataTableUtil.java
Outputting every "loaded" entry in the table is excessive, especially for tables with multiple simultaneous bulk imports and multiple references to the same file. This has been seen to cause performance problems. Even when the log level was reduced, there was blocking within log4j. By doing that check once outside the loop and only logging at trace level, we have seen bulk import performance improvements.
Updated String to byte array for faster comparison
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c31cd49
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c31cd49
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c31cd49
Branch: refs/heads/1.8
Commit: 6c31cd49ce0b8eae13d7a6cca50bc802745a6e72
Parents: d5ddca1
Author: matthpeterson <mp...@appliedtech-group.com>
Authored: Fri Jun 16 15:51:38 2017 -0400
Committer: Michael Wall <mj...@apache.org>
Committed: Tue Jun 20 16:17:38 2017 -0400
----------------------------------------------------------------------
.../accumulo/server/util/MetadataTableUtil.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c31cd49/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index c07114b..54ca37c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -24,6 +24,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -894,12 +895,19 @@ public class MetadataTableUtil {
BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) {
mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ boolean shouldTrace = log.isTraceEnabled();
+ byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8);
for (Entry<Key,Value> entry : mscanner) {
- log.debug("Looking at entry " + entry + " with tid " + tid);
- if (Long.parseLong(entry.getValue().toString()) == tid) {
- log.debug("deleting entry " + entry);
- Mutation m = new Mutation(entry.getKey().getRow());
- m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ if (shouldTrace) {
+ log.trace("Looking at entry {} with tid {}", entry, tid);
+ }
+ if (Arrays.equals(entry.getValue().get(), tidAsBytes)) {
+ if (shouldTrace) {
+ log.trace("deleting entry {}", entry);
+ }
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
bw.addMutation(m);
}
}