You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mj...@apache.org on 2017/06/20 20:42:40 UTC
[1/4] accumulo git commit: ACCUMULO-4657 - backport to 1.7
Repository: accumulo
Updated Branches:
refs/heads/1.8 5194ae723 -> 97c70da13
ACCUMULO-4657 - backport to 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bc557247
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bc557247
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bc557247
Branch: refs/heads/1.8
Commit: bc557247a65cc6dbdcb49e97ed6594d54da547f9
Parents: ede9dae
Author: Michael Wall <mj...@apache.org>
Authored: Tue Jun 20 16:15:21 2017 -0400
Committer: Michael Wall <mj...@apache.org>
Committed: Tue Jun 20 16:15:21 2017 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/util/MetadataTableUtil.java | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc557247/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 0832769..f7fd31f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -23,6 +23,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -908,12 +909,14 @@ public class MetadataTableUtil {
mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8);
for (Entry<Key,Value> entry : mscanner) {
- log.debug("Looking at entry " + entry + " with tid " + tid);
- if (Long.parseLong(entry.getValue().toString()) == tid) {
- log.debug("deleting entry " + entry);
- Mutation m = new Mutation(entry.getKey().getRow());
- m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ log.trace("Looking at entry {} with tid {}", entry, tid);
+ if (Arrays.equals(entry.getValue().get(), tidAsBytes)) {
+ log.trace("deleting entry {}", entry);
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
bw.addMutation(m);
}
}
[4/4] accumulo git commit: ACCUMULO-4657 remove isTraceEnabled guard
Posted by mj...@apache.org.
ACCUMULO-4657 remove isTraceEnabled guard
not needed with slf4j. Closes #270
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/97c70da1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/97c70da1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/97c70da1
Branch: refs/heads/1.8
Commit: 97c70da139de19da1921651181bd38a4660175b8
Parents: 6c31cd4
Author: Michael Wall <mj...@apache.org>
Authored: Tue Jun 20 16:19:38 2017 -0400
Committer: Michael Wall <mj...@apache.org>
Committed: Tue Jun 20 16:42:03 2017 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/MetadataTableUtil.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/97c70da1/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 54ca37c..aac7164 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -895,16 +895,11 @@ public class MetadataTableUtil {
BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) {
mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
- boolean shouldTrace = log.isTraceEnabled();
byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8);
for (Entry<Key,Value> entry : mscanner) {
- if (shouldTrace) {
- log.trace("Looking at entry {} with tid {}", entry, tid);
- }
+ log.trace("Looking at entry {} with tid {}", entry, tid);
if (Arrays.equals(entry.getValue().get(), tidAsBytes)) {
- if (shouldTrace) {
- log.trace("deleting entry {}", entry);
- }
+ log.trace("deleting entry {}", entry);
Key key = entry.getKey();
Mutation m = new Mutation(key.getRow());
m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
[3/4] accumulo git commit: ACCUMULO-4657 Update MetadataTableUtil.java
Posted by mj...@apache.org.
ACCUMULO-4657 Update MetadataTableUtil.java
Outputting every "loaded" entry in the table is excessive, especially for tables with multiple simultaneous bulk imports and multiple references to the same file. This has been seen to cause performance problems. Even when the log level was reduced, there was blocking within log4j. By doing that check once outside the loop and only logging at trace level, we have seen bulk import performance improvements.
Updated String to byte array for faster comparison
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c31cd49
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c31cd49
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c31cd49
Branch: refs/heads/1.8
Commit: 6c31cd49ce0b8eae13d7a6cca50bc802745a6e72
Parents: d5ddca1
Author: matthpeterson <mp...@appliedtech-group.com>
Authored: Fri Jun 16 15:51:38 2017 -0400
Committer: Michael Wall <mj...@apache.org>
Committed: Tue Jun 20 16:17:38 2017 -0400
----------------------------------------------------------------------
.../accumulo/server/util/MetadataTableUtil.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c31cd49/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index c07114b..54ca37c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -24,6 +24,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -894,12 +895,19 @@ public class MetadataTableUtil {
BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) {
mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ boolean shouldTrace = log.isTraceEnabled();
+ byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8);
for (Entry<Key,Value> entry : mscanner) {
- log.debug("Looking at entry " + entry + " with tid " + tid);
- if (Long.parseLong(entry.getValue().toString()) == tid) {
- log.debug("deleting entry " + entry);
- Mutation m = new Mutation(entry.getKey().getRow());
- m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ if (shouldTrace) {
+ log.trace("Looking at entry {} with tid {}", entry, tid);
+ }
+ if (Arrays.equals(entry.getValue().get(), tidAsBytes)) {
+ if (shouldTrace) {
+ log.trace("deleting entry {}", entry);
+ }
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
bw.addMutation(m);
}
}
[2/4] accumulo git commit: Merge branch '1.7' into 1.8
Posted by mj...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d5ddca13
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d5ddca13
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d5ddca13
Branch: refs/heads/1.8
Commit: d5ddca13efa61608550353cbba99a39b6e8ed8dd
Parents: 5194ae7 bc55724
Author: Michael Wall <mj...@apache.org>
Authored: Tue Jun 20 16:16:40 2017 -0400
Committer: Michael Wall <mj...@apache.org>
Committed: Tue Jun 20 16:16:40 2017 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------