You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2018/07/13 20:34:28 UTC
[accumulo] branch master updated (8cc970c -> e374cc5)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from 8cc970c Add filter to listcompactions (#519)
new 5474f5a fixes #512 let iterator know if its a user compaction (#513)
add 8519514 fixes #558 use copy to avoid deadlock in tserver (#559)
new e374cc5 Merge branch '1.9'
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/client/ClientSideIteratorScanner.java | 5 ++
.../core/client/impl/BaseIteratorEnvironment.java | 5 ++
.../accumulo/core/client/impl/OfflineIterator.java | 5 ++
.../core/iterators/IteratorEnvironment.java | 2 +
.../environments/SimpleIteratorEnvironment.java | 5 ++
.../tserver/TabletIteratorEnvironment.java | 14 ++++-
.../apache/accumulo/tserver/tablet/Compactor.java | 4 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 64 ++++++++++++++--------
8 files changed, 79 insertions(+), 25 deletions(-)
[accumulo] 01/02: fixes #512 let iterator know if its a user
compaction (#513)
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5474f5ae021025251e04405de3841ddd1663db15
Author: Brian Buxton <br...@appliedtech-group.com>
AuthorDate: Mon Jun 4 08:12:11 2018 -0400
fixes #512 let iterator know if its a user compaction (#513)
Adjusted interface for IteratorEnvironment and filled in the trivial implementations
Implemented TabletIteratorEnvironment's isUserCompaction
---
.../accumulo/core/client/ClientSideIteratorScanner.java | 5 +++++
.../accumulo/core/client/impl/BaseIteratorEnvironment.java | 5 +++++
.../apache/accumulo/core/client/impl/OfflineIterator.java | 5 +++++
.../accumulo/core/iterators/IteratorEnvironment.java | 2 ++
.../environments/SimpleIteratorEnvironment.java | 5 +++++
.../apache/accumulo/tserver/TabletIteratorEnvironment.java | 14 +++++++++++++-
.../java/org/apache/accumulo/tserver/tablet/Compactor.java | 4 ++--
7 files changed, 37 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index 905d7f7..e0f86d1 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@ -107,6 +107,11 @@ public class ClientSideIteratorScanner extends ScannerOptions implements Scanner
}
@Override
+ public boolean isUserCompaction() {
+ return false;
+ }
+
+ @Override
public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
throw new UnsupportedOperationException();
}
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/BaseIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/client/impl/BaseIteratorEnvironment.java
index ddcb066..f1f811d 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/BaseIteratorEnvironment.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/BaseIteratorEnvironment.java
@@ -57,6 +57,11 @@ public class BaseIteratorEnvironment implements IteratorEnvironment {
}
@Override
+ public boolean isUserCompaction() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
throw new UnsupportedOperationException();
}
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java
index 15de019..2418554 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java
@@ -105,6 +105,11 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
return false;
}
+ @Override
+ public boolean isUserCompaction() {
+ return false;
+ }
+
private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<>();
@Override
diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
index 39a73b6..596c54a 100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
@@ -36,6 +36,8 @@ public interface IteratorEnvironment {
boolean isFullMajorCompaction();
+ boolean isUserCompaction();
+
void registerSideChannel(SortedKeyValueIterator<Key,Value> iter);
Authorizations getAuthorizations();
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java
index 1f6ecf4..ba70501 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java
@@ -54,6 +54,11 @@ public class SimpleIteratorEnvironment implements IteratorEnvironment {
}
@Override
+ public boolean isUserCompaction() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
throw new UnsupportedOperationException();
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
index d3c13b7..c25fde8 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
@@ -35,6 +35,7 @@ import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.tserver.FileManager.ScanFileManager;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
import org.apache.hadoop.fs.Path;
public class TabletIteratorEnvironment implements IteratorEnvironment {
@@ -42,6 +43,7 @@ public class TabletIteratorEnvironment implements IteratorEnvironment {
private final ScanFileManager trm;
private final IteratorScope scope;
private final boolean fullMajorCompaction;
+ private boolean userCompaction;
private final AccumuloConfiguration config;
private final ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators;
private Map<FileRef,DataFileValue> files;
@@ -58,6 +60,7 @@ public class TabletIteratorEnvironment implements IteratorEnvironment {
this.trm = null;
this.config = config;
this.fullMajorCompaction = false;
+ this.userCompaction = false;
this.authorizations = Authorizations.EMPTY;
this.topLevelIterators = new ArrayList<>();
}
@@ -92,7 +95,7 @@ public class TabletIteratorEnvironment implements IteratorEnvironment {
}
public TabletIteratorEnvironment(IteratorScope scope, boolean fullMajC,
- AccumuloConfiguration config) {
+ AccumuloConfiguration config, MajorCompactionReason reason) {
if (scope != IteratorScope.majc)
throw new IllegalArgumentException(
"Tried to set maj compaction type when scope was " + scope);
@@ -101,6 +104,7 @@ public class TabletIteratorEnvironment implements IteratorEnvironment {
this.trm = null;
this.config = config;
this.fullMajorCompaction = fullMajC;
+ this.userCompaction = reason.equals(MajorCompactionReason.USER);
this.authorizations = Authorizations.EMPTY;
this.topLevelIterators = new ArrayList<>();
}
@@ -123,6 +127,14 @@ public class TabletIteratorEnvironment implements IteratorEnvironment {
}
@Override
+ public boolean isUserCompaction() {
+ if (scope != IteratorScope.majc)
+ throw new IllegalStateException(
+ "Asked about user initiated compaction type when scope is " + scope);
+ return userCompaction;
+ }
+
+ @Override
public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName)
throws IOException {
FileRef ref = new FileRef(mapFileName, new Path(mapFileName));
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index f31bb65..a525a4a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -351,8 +351,8 @@ public class Compactor implements Callable<CompactionStats> {
TabletIteratorEnvironment iterEnv;
if (env.getIteratorScope() == IteratorScope.majc)
- iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes,
- acuTableConf);
+ iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf,
+ getMajorCompactionReason());
else if (env.getIteratorScope() == IteratorScope.minc)
iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
else
[accumulo] 02/02: Merge branch '1.9'
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e374cc513ee13edbcb745ce04ebab34ad523eba2
Merge: 5474f5a 8519514
Author: Keith Turner <kt...@apache.org>
AuthorDate: Fri Jul 13 16:34:03 2018 -0400
Merge branch '1.9'
.../org/apache/accumulo/tserver/tablet/Tablet.java | 64 ++++++++++++++--------
1 file changed, 42 insertions(+), 22 deletions(-)
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 013c5c7,4dc19f2..b2733ca
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -147,14 -148,17 +147,17 @@@ import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
+ import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+ import com.google.common.collect.ImmutableSet;
+ import com.google.common.collect.ImmutableSet.Builder;
/**
*
@@@ -494,9 -498,11 +497,11 @@@ public class Tablet implements TabletCo
logEntry.getColumnQualifier().toString()));
}
+ rebuildReferencedLogs();
+
log.info(
- "Write-Ahead Log recovery complete for " + this.extent + " (" + entriesUsedOnTablet.get()
- + " mutations applied, " + getTabletMemory().getNumEntries() + " entries created)");
+ "Write-Ahead Log recovery complete for {} ({} mutations applied, {} entries created)",
+ this.extent, entriesUsedOnTablet.get(), getTabletMemory().getNumEntries());
}
String contextName = tableConfiguration.get(Property.TABLE_CLASSPATH);
@@@ -2480,16 -2519,14 +2499,14 @@@
}
for (String logger : currentLogsCopy) {
- log.debug("Logs for current memory: " + getExtent() + " " + logger);
+ log.debug("Logs for current memory: {} {}", getExtent(), logger);
}
- Set<String> doomed = new HashSet<>();
- for (DfsLogger logger : doomedLogs) {
- log.debug("Logs to be destroyed: {} {}", getExtent(), logger.getMeta());
- doomed.add(logger.getMeta());
+ for (String logger : unusedLogs) {
- log.debug("Logs to be destroyed: " + getExtent() + " " + logger);
++ log.debug("Logs to be destroyed: {} {}", getExtent(), logger);
}
- return doomed;
+ return unusedLogs;
}
synchronized void finishClearingUnusedLogs() {