You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/09/24 00:34:03 UTC
[03/47] hbase git commit: HBASE-18453 CompactionRequest should not be
exposed to user directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
index 45b7d3c..4890f0d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
@@ -23,10 +23,12 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CompareOperator;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -41,13 +43,17 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.Service;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.Service;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall;
/**
* Regions store data for a certain region of a table. It stores all columns
@@ -105,7 +111,7 @@ public interface Region extends ConfigurationObserver {
* <p>Use with caution. Exposed for use of fixup utilities.
* @return a list of the Stores managed by this region
*/
- List<Store> getStores();
+ List<? extends Store> getStores();
/**
* Return the Store for the given family
@@ -115,7 +121,7 @@ public interface Region extends ConfigurationObserver {
Store getStore(byte[] family);
/** @return list of store file names for the given families */
- List<String> getStoreFileList(byte [][] columns);
+ List<String> getStoreFileList(byte[][] columns);
/**
* Check the region's underlying store files, open the files that have not
@@ -753,6 +759,18 @@ public interface Region extends ConfigurationObserver {
*/
CompactionState getCompactionState();
+ /**
+ * Request compaction on this region.
+ */
+ void requestCompaction(String why, int priority, CompactionLifeCycleTracker tracker, User user)
+ throws IOException;
+
+ /**
+ * Request compaction for the given family
+ */
+ void requestCompaction(byte[] family, String why, int priority,
+ CompactionLifeCycleTracker tracker, User user) throws IOException;
+
/** Wait for all current flushes and compactions of the region to complete */
void waitForFlushesAndCompactions();
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index bed2a7a..f282766 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
import org.apache.hadoop.hbase.regionserver.Region.Operation;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker;
import org.apache.hadoop.hbase.security.User;
@@ -499,18 +500,18 @@ public class RegionCoprocessorHost
/**
* See
* {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long,
- * InternalScanner, CompactionRequest, long)}
+ * InternalScanner, CompactionLifeCycleTracker, long)}
*/
- public InternalScanner preCompactScannerOpen(final Store store,
- final List<StoreFileScanner> scanners, final ScanType scanType, final long earliestPutTs,
- final CompactionRequest request, final User user, final long readPoint) throws IOException {
+ public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners,
+ ScanType scanType, long earliestPutTs, CompactionLifeCycleTracker tracker, User user,
+ long readPoint) throws IOException {
return execOperationWithResult(null,
coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>(user) {
@Override
public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
throws IOException {
setResult(oserver.preCompactScannerOpen(ctx, store, scanners, scanType,
- earliestPutTs, getResult(), request, readPoint));
+ earliestPutTs, getResult(), tracker, readPoint));
}
});
}
@@ -520,17 +521,17 @@ public class RegionCoprocessorHost
* available candidates.
* @param store The store where compaction is being requested
* @param candidates The currently available store files
- * @param request custom compaction request
+ * @param tracker used to track the life cycle of a compaction
* @return If {@code true}, skip the normal selection process and use the current list
* @throws IOException
*/
- public boolean preCompactSelection(final Store store, final List<StoreFile> candidates,
- final CompactionRequest request, final User user) throws IOException {
+ public boolean preCompactSelection(Store store, List<StoreFile> candidates,
+ CompactionLifeCycleTracker tracker, User user) throws IOException {
return execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) {
@Override
public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
throws IOException {
- oserver.preCompactSelection(ctx, store, candidates, request);
+ oserver.preCompactSelection(ctx, store, candidates, tracker);
}
});
}
@@ -540,21 +541,17 @@ public class RegionCoprocessorHost
* candidates.
* @param store The store where compaction is being requested
* @param selected The store files selected to compact
- * @param request custom compaction
+ * @param tracker used to track the life cycle of a compaction
*/
- public void postCompactSelection(final Store store, final ImmutableList<StoreFile> selected,
- final CompactionRequest request, final User user) {
- try {
- execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) {
- @Override
- public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
- throws IOException {
- oserver.postCompactSelection(ctx, store, selected, request);
- }
- });
- } catch (IOException e) {
- LOG.warn(e);
- }
+ public void postCompactSelection(Store store, ImmutableList<StoreFile> selected,
+ CompactionLifeCycleTracker tracker, User user) throws IOException {
+ execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) {
+ @Override
+ public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
+ throws IOException {
+ oserver.postCompactSelection(ctx, store, selected, tracker);
+ }
+ });
}
/**
@@ -562,18 +559,17 @@ public class RegionCoprocessorHost
* @param store the store being compacted
* @param scanner the scanner used to read store data during compaction
* @param scanType type of Scan
- * @param request the compaction that will be executed
+ * @param tracker used to track the life cycle of a compaction
* @throws IOException
*/
- public InternalScanner preCompact(final Store store, final InternalScanner scanner,
- final ScanType scanType, final CompactionRequest request, final User user)
- throws IOException {
+ public InternalScanner preCompact(Store store, InternalScanner scanner, ScanType scanType,
+ CompactionLifeCycleTracker tracker, User user) throws IOException {
return execOperationWithResult(false, scanner,
coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>(user) {
@Override
public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
throws IOException {
- setResult(oserver.preCompact(ctx, store, getResult(), scanType, request));
+ setResult(oserver.preCompact(ctx, store, getResult(), scanType, tracker));
}
});
}
@@ -582,16 +578,16 @@ public class RegionCoprocessorHost
* Called after the store compaction has completed.
* @param store the store being compacted
* @param resultFile the new store file written during compaction
- * @param request the compaction that is being executed
+ * @param tracker used to track the life cycle of a compaction
* @throws IOException
*/
- public void postCompact(final Store store, final StoreFile resultFile,
- final CompactionRequest request, final User user) throws IOException {
+ public void postCompact(Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker,
+ User user) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) {
@Override
public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
throws IOException {
- oserver.postCompact(ctx, store, resultFile, request);
+ oserver.postCompact(ctx, store, resultFile, tracker);
}
});
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index b56c925..209fce8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -64,11 +64,6 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi
List<WAL> getWALs() throws IOException;
/**
- * @return Implementation of {@link CompactionRequestor} or null.
- */
- CompactionRequestor getCompactionRequester();
-
- /**
* @return Implementation of {@link FlushRequester} or null.
*/
FlushRequester getFlushRequester();
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
index 7281626..5ccd6e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
@@ -76,7 +76,7 @@ public abstract class RegionSplitPolicy extends Configured {
if (explicitSplitPoint != null) {
return explicitSplitPoint;
}
- List<Store> stores = region.getStores();
+ List<HStore> stores = region.getStores();
byte[] splitPointFromLargestStore = null;
long largestStoreSize = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 4bb31ae..b7e83bf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.NavigableSet;
+import java.util.Optional;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.CellComparator;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher;
@@ -51,7 +53,8 @@ import org.apache.hadoop.hbase.security.User;
@InterfaceStability.Evolving
public interface Store extends HeapSize, StoreConfigInformation, PropagatingConfigurationObserver {
- /* The default priority for user-specified compaction requests.
+ /**
+ * The default priority for user-specified compaction requests.
* The user gets top priority unless we have blocking compactions. (Pri <= 0)
*/
int PRIORITY_USER = 1;
@@ -253,17 +256,12 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
*/
CompactionProgress getCompactionProgress();
- CompactionContext requestCompaction() throws IOException;
-
- /**
- * @deprecated see requestCompaction(int, CompactionRequest, User)
- */
- @Deprecated
- CompactionContext requestCompaction(int priority, CompactionRequest baseRequest)
- throws IOException;
+ default Optional<CompactionContext> requestCompaction() throws IOException {
+ return requestCompaction(NO_PRIORITY, CompactionLifeCycleTracker.DUMMY, null);
+ }
- CompactionContext requestCompaction(int priority, CompactionRequest baseRequest, User user)
- throws IOException;
+ Optional<CompactionContext> requestCompaction(int priority, CompactionLifeCycleTracker tracker,
+ User user) throws IOException;
void cancelRequestedCompaction(CompactionContext compaction);
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java
new file mode 100644
index 0000000..38fec7e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.compactions;
+
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+
+/**
+ * Used to track compaction execution.
+ */
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
+public interface CompactionLifeCycleTracker {
+
+ static CompactionLifeCycleTracker DUMMY = new CompactionLifeCycleTracker() {
+ };
+
+ /**
+ * Called before compaction is executed by CompactSplitThread.
+ * <p>
+ * Requesting compaction on a region can lead to multiple compactions on different stores, so we
+ * will pass the {@link Store} in to tell you the store we operate on.
+ */
+ default void beforeExecute(Store store) {
+ }
+
+ /**
+ * Called after compaction is executed by CompactSplitThread.
+ * <p>
+ * Requesting compaction on a region can lead to multiple compactions on different stores, so we
+ * will pass the {@link Store} in to tell you the store we operate on.
+ */
+ default void afterExecute(Store store) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
index 69e354b..e05c165 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
@@ -18,25 +18,21 @@
*/
package org.apache.hadoop.hbase.regionserver.compactions;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.stream.Collectors;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFileReader;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
/**
* This class holds all logical details necessary to run a compaction.
*/
-@InterfaceAudience.LimitedPrivate({ "coprocessor" })
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
public class CompactionRequest {
// was this compaction promoted to an off-peak
@@ -53,56 +49,18 @@ public class CompactionRequest {
private String regionName = "";
private String storeName = "";
private long totalSize = -1L;
+ private CompactionLifeCycleTracker tracker = CompactionLifeCycleTracker.DUMMY;
- /**
- * This ctor should be used by coprocessors that want to subclass CompactionRequest.
- */
- public CompactionRequest() {
+ public CompactionRequest(Collection<StoreFile> files) {
this.selectionTime = EnvironmentEdgeManager.currentTime();
this.timeInNanos = System.nanoTime();
- }
-
- public CompactionRequest(Collection<StoreFile> files) {
- this();
- Preconditions.checkNotNull(files);
- this.filesToCompact = files;
+ this.filesToCompact = Preconditions.checkNotNull(files, "files for compaction can not null");
recalculateSize();
}
public void updateFiles(Collection<StoreFile> files) {
- this.filesToCompact = files;
- recalculateSize();
- }
-
- /**
- * Called before compaction is executed by CompactSplitThread; for use by coproc subclasses.
- */
- public void beforeExecute() {}
-
- /**
- * Called after compaction is executed by CompactSplitThread; for use by coproc subclasses.
- */
- public void afterExecute() {}
-
- /**
- * Combines the request with other request. Coprocessors subclassing CR may override
- * this if they want to do clever things based on CompactionPolicy selection that
- * is passed to this method via "other". The default implementation just does a copy.
- * @param other Request to combine with.
- * @return The result (may be "this" or "other").
- */
- public CompactionRequest combineWith(CompactionRequest other) {
- this.filesToCompact = new ArrayList<>(other.getFiles());
- this.isOffPeak = other.isOffPeak;
- this.isMajor = other.isMajor;
- this.priority = other.priority;
- this.selectionTime = other.selectionTime;
- this.timeInNanos = other.timeInNanos;
- this.regionName = other.regionName;
- this.storeName = other.storeName;
- this.totalSize = other.totalSize;
+ this.filesToCompact = Preconditions.checkNotNull(files, "files for compaction can not null");
recalculateSize();
- return this;
}
public Collection<StoreFile> getFiles() {
@@ -168,6 +126,14 @@ public class CompactionRequest {
: (isMajor ? DisplayCompactionType.MAJOR : DisplayCompactionType.ALL_FILES);
}
+ public void setTracker(CompactionLifeCycleTracker tracker) {
+ this.tracker = tracker;
+ }
+
+ public CompactionLifeCycleTracker getTracker() {
+ return tracker;
+ }
+
@Override
public String toString() {
String fsList = filesToCompact.stream().filter(f -> f.getReader() != null)
@@ -186,12 +152,7 @@ public class CompactionRequest {
* @param files files that should be included in the compaction
*/
private void recalculateSize() {
- long sz = 0;
- for (StoreFile sf : this.filesToCompact) {
- StoreFileReader r = sf.getReader();
- sz += r == null ? 0 : r.length();
- }
- this.totalSize = sz;
+ this.totalSize = filesToCompact.stream().map(StoreFile::getReader)
+ .mapToLong(r -> r != null ? r.length() : 0L).sum();
}
}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 0ca925e..e6d1935 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -338,14 +338,14 @@ public abstract class Compactor<T extends CellSink> {
* @param readPoint the read point to help create scanner by Coprocessor if required.
* @return Scanner override by coprocessor; null if not overriding.
*/
- protected InternalScanner preCreateCoprocScanner(final CompactionRequest request,
- final ScanType scanType, final long earliestPutTs, final List<StoreFileScanner> scanners,
- User user, final long readPoint) throws IOException {
+ protected InternalScanner preCreateCoprocScanner(CompactionRequest request, ScanType scanType,
+ long earliestPutTs, List<StoreFileScanner> scanners, User user, long readPoint)
+ throws IOException {
if (store.getCoprocessorHost() == null) {
return null;
}
return store.getCoprocessorHost().preCompactScannerOpen(store, scanners, scanType,
- earliestPutTs, request, user, readPoint);
+ earliestPutTs, request.getTracker(), user, readPoint);
}
/**
@@ -355,12 +355,13 @@ public abstract class Compactor<T extends CellSink> {
* @param scanner The default scanner created for compaction.
* @return Scanner scanner to use (usually the default); null if compaction should not proceed.
*/
- protected InternalScanner postCreateCoprocScanner(final CompactionRequest request,
- final ScanType scanType, final InternalScanner scanner, User user) throws IOException {
+ protected InternalScanner postCreateCoprocScanner(CompactionRequest request, ScanType scanType,
+ InternalScanner scanner, User user) throws IOException {
if (store.getCoprocessorHost() == null) {
return scanner;
}
- return store.getCoprocessorHost().preCompact(store, scanner, scanType, request, user);
+ return store.getCoprocessorHost().preCompact(store, scanner, scanType, request.getTracker(),
+ user);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index f7d70d2..c34fc6d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
@@ -1532,7 +1533,8 @@ public class AccessController implements MasterObserver, RegionObserver, RegionS
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException {
+ InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker)
+ throws IOException {
requirePermission(getActiveUser(c), "compact", getTableName(c.getEnvironment()), null, null,
Action.ADMIN, Action.CREATE);
return scanner;
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp
index c1e5aae..04aafa7 100644
--- a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp
@@ -92,7 +92,7 @@
</div>
<% if(region != null) { //
- List<Store> stores = region.getStores();
+ List<? extends Store> stores = region.getStores();
for (Store store : stores) {
String cf = store.getColumnFamilyName();
Collection<StoreFile> storeFiles = store.getStorefiles(); %>
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index a99345b..76580f1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase;
+import com.google.protobuf.Service;
+
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collections;
@@ -36,11 +38,8 @@ import org.apache.hadoop.hbase.client.locking.EntityLock;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
-import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
import org.apache.hadoop.hbase.regionserver.FlushRequester;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
import org.apache.hadoop.hbase.regionserver.Leases;
@@ -51,14 +50,14 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager;
import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
-import com.google.protobuf.Service;
-
/**
* Basic mock region server services. Should only be instantiated by HBaseTestingUtility.b
*/
@@ -160,11 +159,6 @@ public class MockRegionServerServices implements RegionServerServices {
}
@Override
- public CompactionRequestor getCompactionRequester() {
- return null;
- }
-
- @Override
public ClusterConnection getConnection() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
index 94d1cf6..85f65e8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
@@ -124,7 +124,7 @@ public class TestIOFencing {
}
@Override
- public boolean compact(CompactionContext compaction, Store store,
+ public boolean compact(CompactionContext compaction, HStore store,
ThroughputController throughputController) throws IOException {
try {
return super.compact(compaction, store, throughputController);
@@ -134,7 +134,7 @@ public class TestIOFencing {
}
@Override
- public boolean compact(CompactionContext compaction, Store store,
+ public boolean compact(CompactionContext compaction, HStore store,
ThroughputController throughputController, User user) throws IOException {
try {
return super.compact(compaction, store, throughputController, user);
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index aff3d99..ac404bb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -256,7 +256,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
- InternalScanner s, CompactionRequest request, long readPoint) throws IOException {
+ InternalScanner s, CompactionLifeCycleTracker request, long readPoint) throws IOException {
return createCompactorScanner(store, scanners, scanType, earliestPutTs);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
index d5f3358..08ec09b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
@@ -784,7 +784,7 @@ public class TestBlockEvictionFromClient {
}
private BlockCache setCacheProperties(Region region) {
- Iterator<Store> strItr = region.getStores().iterator();
+ Iterator<? extends Store> strItr = region.getStores().iterator();
BlockCache cache = null;
while (strItr.hasNext()) {
Store store = strItr.next();
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index bef6f6b..023f8fc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileReader;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -203,20 +204,20 @@ public class SimpleRegionObserver implements RegionObserver {
@Override
public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<StoreFile> candidates, CompactionRequest request) throws IOException {
+ List<StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException {
ctPreCompactSelect.incrementAndGet();
}
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- ImmutableList<StoreFile> selected, CompactionRequest request) {
+ ImmutableList<StoreFile> selected, CompactionLifeCycleTracker tracker) {
ctPostCompactSelect.incrementAndGet();
}
-
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException {
+ InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker)
+ throws IOException {
ctPreCompact.incrementAndGet();
return scanner;
}
@@ -224,14 +225,14 @@ public class SimpleRegionObserver implements RegionObserver {
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
- InternalScanner s, CompactionRequest request, long readPoint) throws IOException {
+ InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException {
ctPreCompactScanner.incrementAndGet();
return s;
}
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- StoreFile resultFile, CompactionRequest request) throws IOException {
+ StoreFile resultFile, CompactionLifeCycleTracker tracker) throws IOException {
ctPostCompact.incrementAndGet();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index 97c45ab..5cf0bb3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -194,13 +195,13 @@ public class TestCoprocessorInterface {
}
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
- Store store, InternalScanner scanner, ScanType scanType, CompactionRequest request) {
+ Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) {
preCompactCalled = true;
return scanner;
}
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
- Store store, StoreFile resultFile, CompactionRequest request) {
+ Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker) {
postCompactCalled = true;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index 07fc179..0641b56 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -71,7 +71,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
@@ -417,7 +417,7 @@ public class TestRegionObserverInterface {
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
- final InternalScanner scanner, final ScanType scanType, CompactionRequest request) {
+ InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) {
return new InternalScanner() {
@Override
public boolean next(List<Cell> results) throws IOException {
@@ -456,7 +456,7 @@ public class TestRegionObserverInterface {
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
- StoreFile resultFile, CompactionRequest request) {
+ StoreFile resultFile, CompactionLifeCycleTracker tracker) {
lastCompaction = EnvironmentEdgeManager.currentTime();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index 8ab0175..9c06c3e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.regionserver.ChunkCreator;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl;
@@ -61,6 +62,7 @@ import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.security.User;
@@ -155,7 +157,7 @@ public class TestRegionObserverScannerOpenHook {
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
- long earliestPutTs, InternalScanner s, CompactionRequest request, long readPoint)
+ long earliestPutTs, InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint)
throws IOException {
scanners.forEach(KeyValueScanner::close);
return NO_DATA;
@@ -252,7 +254,7 @@ public class TestRegionObserverScannerOpenHook {
}
@Override
- public boolean compact(CompactionContext compaction, Store store,
+ public boolean compact(CompactionContext compaction, HStore store,
ThroughputController throughputController) throws IOException {
boolean ret = super.compact(compaction, store, throughputController);
if (ret) compactionStateChangeLatch.countDown();
@@ -260,7 +262,7 @@ public class TestRegionObserverScannerOpenHook {
}
@Override
- public boolean compact(CompactionContext compaction, Store store,
+ public boolean compact(CompactionContext compaction, HStore store,
ThroughputController throughputController, User user) throws IOException {
boolean ret = super.compact(compaction, store, throughputController, user);
if (ret) compactionStateChangeLatch.countDown();
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 3ffa61b..ba18299 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -47,6 +47,27 @@ import org.apache.hadoop.hbase.client.locking.EntityLock;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
+import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager;
+import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
+import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
@@ -61,10 +82,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegion
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
@@ -102,27 +123,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRespon
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
-import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
-import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
-import org.apache.hadoop.hbase.regionserver.FlushRequester;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
-import org.apache.hadoop.hbase.regionserver.Leases;
-import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager;
-import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
-import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
/**
* A mock RegionServer implementation.
@@ -315,12 +315,6 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
- public CompactionRequestor getCompactionRequester() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
public FlushRequester getFlushRequester() {
// TODO Auto-generated method stub
return null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
index 398d14d..12aed50 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -86,7 +86,7 @@ import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -716,14 +716,14 @@ public class TestMobCompactor {
}
/**
- * This copro overwrites the default compaction policy. It always chooses two latest
- * hfiles and compacts them into a new one.
+ * This copro overwrites the default compaction policy. It always chooses two latest hfiles and
+ * compacts them into a new one.
*/
public static class CompactTwoLatestHfilesCopro implements RegionObserver {
+
@Override
- public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Store store, final List<StoreFile> candidates, final CompactionRequest request)
- throws IOException {
+ public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ List<StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException {
int count = candidates.size();
if (count >= 2) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 5179b84..8015115 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.quotas.QuotaUtil;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -461,7 +462,7 @@ public class TestNamespaceAuditor {
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
- StoreFile resultFile, CompactionRequest request) throws IOException {
+ StoreFile resultFile, CompactionLifeCycleTracker tracker) throws IOException {
postCompact.countDown();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java
index 823b1f7..0d07e1a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -324,7 +324,7 @@ public class TestFileSystemUtilizationChore {
final HRegionInfo info = mock(HRegionInfo.class);
when(r.getRegionInfo()).thenReturn(info);
List<Store> stores = new ArrayList<>();
- when(r.getStores()).thenReturn(stores);
+ when(r.getStores()).thenReturn((List) stores);
for (Long storeSize : storeSizes) {
final Store s = mock(Store.class);
stores.add(s);
@@ -338,7 +338,7 @@ public class TestFileSystemUtilizationChore {
final HRegionInfo info = mock(HRegionInfo.class);
when(r.getRegionInfo()).thenReturn(info);
List<Store> stores = new ArrayList<>();
- when(r.getStores()).thenReturn(stores);
+ when(r.getStores()).thenReturn((List) stores);
assertEquals(
"Logic error, storeSizes and linkSizes must be equal in size", storeSizes.size(),
hfileSizes.size());
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
index fcc3d4a..36c2e19 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.TestFromClientSideWithCoprocessor;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
/**
@@ -61,7 +62,7 @@ public class NoOpScanPolicyObserver implements RegionObserver {
public InternalScanner preCompactScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c, Store store,
List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
- InternalScanner s, CompactionRequest request, long readPoint) throws IOException {
+ InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException {
// this demonstrates how to override the scanners default behavior
ScanInfo oldSI = store.getScanInfo();
ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(),
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
index 0526462..a1fe87b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
@@ -18,11 +18,16 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import static org.mockito.Matchers.*;
-import static org.mockito.Mockito.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Optional;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.security.User;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -33,15 +38,23 @@ import org.mockito.stubbing.Answer;
*/
public class StatefulStoreMockMaker {
// Add and expand the methods and answers as needed.
- public CompactionContext selectCompaction() { return null; }
- public void cancelCompaction(Object originalContext) {}
- public int getPriority() { return 0; }
+ public Optional<CompactionContext> selectCompaction() {
+ return Optional.empty();
+ }
- private class SelectAnswer implements Answer<CompactionContext> {
- public CompactionContext answer(InvocationOnMock invocation) throws Throwable {
+ public void cancelCompaction(Object originalContext) {
+ }
+
+ public int getPriority() {
+ return 0;
+ }
+
+ private class SelectAnswer implements Answer<Optional<CompactionContext>> {
+ public Optional<CompactionContext> answer(InvocationOnMock invocation) throws Throwable {
return selectCompaction();
}
}
+
private class PriorityAnswer implements Answer<Integer> {
public Integer answer(InvocationOnMock invocation) throws Throwable {
return getPriority();
@@ -53,15 +66,13 @@ public class StatefulStoreMockMaker {
}
}
- public Store createStoreMock(String name) throws Exception {
- Store store = mock(Store.class, name);
- when(store.requestCompaction(
- anyInt(), isNull(CompactionRequest.class))).then(new SelectAnswer());
- when(store.requestCompaction(
- anyInt(), isNull(CompactionRequest.class), any(User.class))).then(new SelectAnswer());
+ public HStore createStoreMock(String name) throws Exception {
+ HStore store = mock(HStore.class, name);
+ when(store.requestCompaction(anyInt(), any(CompactionLifeCycleTracker.class), any(User.class)))
+ .then(new SelectAnswer());
when(store.getCompactPriority()).then(new PriorityAnswer());
- doAnswer(new CancelAnswer()).when(
- store).cancelRequestedCompaction(any(CompactionContext.class));
+ doAnswer(new CancelAnswer()).when(store)
+ .cancelRequestedCompaction(any(CompactionContext.class));
return store;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index be078f2..3649823 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -33,8 +33,8 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
@@ -65,7 +66,6 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL;
import org.junit.After;
@@ -298,15 +298,16 @@ public class TestCompaction {
Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
// setup a region/store with some files
- Store store = r.getStore(COLUMN_FAMILY);
+ HStore store = r.getStore(COLUMN_FAMILY);
createStoreFile(r);
for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
createStoreFile(r);
}
CountDownLatch latch = new CountDownLatch(1);
- TrackableCompactionRequest request = new TrackableCompactionRequest(latch);
- thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request,null);
+ Tracker tracker = new Tracker(latch);
+ thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, tracker,
+ null);
// wait for the latch to complete.
latch.await();
@@ -322,7 +323,7 @@ public class TestCompaction {
Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
// setup a region/store with some files
- Store store = r.getStore(COLUMN_FAMILY);
+ HStore store = r.getStore(COLUMN_FAMILY);
createStoreFile(r);
for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) {
createStoreFile(r);
@@ -337,9 +338,9 @@ public class TestCompaction {
long preFailedCount = metricsWrapper.getNumCompactionsFailed();
CountDownLatch latch = new CountDownLatch(1);
- TrackableCompactionRequest request = new TrackableCompactionRequest(latch);
+ Tracker tracker = new Tracker(latch);
thread.requestCompaction(mockRegion, store, "test custom comapction", Store.PRIORITY_USER,
- request, null);
+ tracker, null);
// wait for the latch to complete.
latch.await(120, TimeUnit.SECONDS);
@@ -370,20 +371,17 @@ public class TestCompaction {
// setup a region/store with some files
int numStores = r.getStores().size();
- List<Pair<CompactionRequest, Store>> requests = new ArrayList<>(numStores);
CountDownLatch latch = new CountDownLatch(numStores);
+ Tracker tracker = new Tracker(latch);
// create some store files and setup requests for each store on which we want to do a
// compaction
- for (Store store : r.getStores()) {
+ for (HStore store : r.getStores()) {
createStoreFile(r, store.getColumnFamilyName());
createStoreFile(r, store.getColumnFamilyName());
createStoreFile(r, store.getColumnFamilyName());
- requests.add(new Pair<>(new TrackableCompactionRequest(latch), store));
+ thread.requestCompaction(r, store, "test mulitple custom comapctions", Store.PRIORITY_USER,
+ tracker, null);
}
-
- thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER,
- Collections.unmodifiableList(requests), null);
-
// wait for the latch to complete.
latch.await();
@@ -428,7 +426,7 @@ public class TestCompaction {
}
@Override
- public synchronized CompactionContext selectCompaction() {
+ public synchronized Optional<CompactionContext> selectCompaction() {
CompactionContext ctx = new TestCompactionContext(new ArrayList<>(notCompacting));
compacting.addAll(notCompacting);
notCompacting.clear();
@@ -437,7 +435,7 @@ public class TestCompaction {
} catch (IOException ex) {
fail("Shouldn't happen");
}
- return ctx;
+ return Optional.of(ctx);
}
@Override
@@ -499,14 +497,14 @@ public class TestCompaction {
}
@Override
- public CompactionContext selectCompaction() {
+ public Optional<CompactionContext> selectCompaction() {
this.blocked = new BlockingCompactionContext();
try {
this.blocked.select(null, false, false, false);
} catch (IOException ex) {
fail("Shouldn't happen");
}
- return this.blocked;
+ return Optional.of(blocked);
}
@Override
@@ -527,13 +525,13 @@ public class TestCompaction {
}
@Override
- public Store createStoreMock(String name) throws Exception {
+ public HStore createStoreMock(String name) throws Exception {
return createStoreMock(Integer.MIN_VALUE, name);
}
- public Store createStoreMock(int priority, String name) throws Exception {
+ public HStore createStoreMock(int priority, String name) throws Exception {
// Override the mock to always return the specified priority.
- Store s = super.createStoreMock(name);
+ HStore s = super.createStoreMock(name);
when(s.getCompactPriority()).thenReturn(priority);
return s;
}
@@ -555,7 +553,7 @@ public class TestCompaction {
// Set up the region mock that redirects compactions.
HRegion r = mock(HRegion.class);
when(
- r.compact(any(CompactionContext.class), any(Store.class),
+ r.compact(any(CompactionContext.class), any(HStore.class),
any(ThroughputController.class), any(User.class))).then(new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocation) throws Throwable {
@@ -568,7 +566,7 @@ public class TestCompaction {
// Set up store mocks for 2 "real" stores and the one we use for blocking CST.
ArrayList<Integer> results = new ArrayList<>();
StoreMockMaker sm = new StoreMockMaker(results), sm2 = new StoreMockMaker(results);
- Store store = sm.createStoreMock("store1"), store2 = sm2.createStoreMock("store2");
+ HStore store = sm.createStoreMock("store1"), store2 = sm2.createStoreMock("store2");
BlockingStoreMockMaker blocker = new BlockingStoreMockMaker();
// First, block the compaction thread so that we could muck with queue.
@@ -691,24 +689,20 @@ public class TestCompaction {
}
/**
- * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes.
+ * Simple {@link CompactionLifeCycleTracker} on which you can wait until the requested compaction
+ * finishes.
*/
- public static class TrackableCompactionRequest extends CompactionRequest {
- private CountDownLatch done;
-
- /**
- * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
- * compaction before being used.
- */
- public TrackableCompactionRequest(CountDownLatch finished) {
- super();
- this.done = finished;
+ public static class Tracker implements CompactionLifeCycleTracker {
+
+ private final CountDownLatch done;
+
+ public Tracker(CountDownLatch done) {
+ this.done = done;
}
@Override
- public void afterExecute() {
- super.afterExecute();
- this.done.countDown();
+ public void afterExecute(Store store) {
+ done.countDown();
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
index 2cea121..603203a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -28,6 +28,7 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableSet;
+import java.util.Optional;
import java.util.concurrent.ConcurrentSkipListSet;
import javax.crypto.spec.SecretKeySpec;
@@ -60,6 +61,7 @@ import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User;
@@ -536,8 +538,9 @@ public class TestHMobStore {
// Trigger major compaction
this.store.triggerMajorCompaction();
- CompactionContext requestCompaction = this.store.requestCompaction(1, null);
- this.store.compact(requestCompaction, NoLimitThroughputController.INSTANCE, null);
+ Optional<CompactionContext> requestCompaction =
+ this.store.requestCompaction(Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
+ this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
Assert.assertEquals(1, this.store.getStorefiles().size());
//Check encryption after compaction
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
index ea14962..97f8ce3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.wal.TestWALActionsListener;
import org.apache.hadoop.hbase.wal.WALEdit;
@@ -254,7 +255,8 @@ public class TestHRegionServerBulkLoad {
static int sleepDuration;
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
- InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException {
+ InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker)
+ throws IOException {
try {
Thread.sleep(sleepDuration);
} catch (InterruptedException ie) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index f45c76c..707540a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
@@ -417,7 +418,7 @@ public class TestMajorCompaction {
}
store.triggerMajorCompaction();
- CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null).getRequest();
+ CompactionRequest request = store.requestCompaction().get().getRequest();
assertNotNull("Expected to receive a compaction request", request);
assertEquals(
"System-requested major compaction should not occur if there are too many store files",
@@ -436,7 +437,9 @@ public class TestMajorCompaction {
createStoreFile(r);
}
store.triggerMajorCompaction();
- CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null).getRequest();
+ CompactionRequest request =
+ store.requestCompaction(Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get()
+ .getRequest();
assertNotNull("Expected to receive a compaction request", request);
assertEquals(
"User-requested major compaction should always occur, even if there are too many store files",
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 69bc9a7..3ba2299 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -31,6 +31,7 @@ import java.io.InterruptedIOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -70,7 +71,11 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
-import org.apache.hadoop.hbase.master.*;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
+import org.apache.hadoop.hbase.master.NoSuchProcedureException;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
@@ -234,7 +239,7 @@ public class TestSplitTransactionOnCluster {
assertEquals(1, cluster.getRegions(tableName).size());
HRegion region = cluster.getRegions(tableName).get(0);
- Store store = region.getStore(cf);
+ HStore store = region.getStore(cf);
int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
@@ -246,8 +251,8 @@ public class TestSplitTransactionOnCluster {
int fileNum = store.getStorefiles().size();
// 0, Compaction Request
store.triggerMajorCompaction();
- CompactionContext cc = store.requestCompaction();
- assertNotNull(cc);
+ Optional<CompactionContext> cc = store.requestCompaction();
+ assertTrue(cc.isPresent());
// 1, A timeout split
// 1.1 close region
assertEquals(2, region.close(false).get(cf).size());
@@ -255,7 +260,7 @@ public class TestSplitTransactionOnCluster {
region.initialize();
// 2, Run Compaction cc
- assertFalse(region.compact(cc, store, NoLimitThroughputController.INSTANCE));
+ assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
assertTrue(fileNum > store.getStorefiles().size());
// 3, Split
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java
index c959a22..78c5330 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java
@@ -117,7 +117,7 @@ public class TestSplitWalDataLoss {
}
}).when(spiedRegion).internalFlushCacheAndCommit(Matchers.<WAL> any(),
Matchers.<MonitoredTask> any(), Matchers.<PrepareFlushResult> any(),
- Matchers.<Collection<Store>> any());
+ Matchers.<Collection<HStore>> any());
// Find region key; don't pick up key for hbase:meta by mistake.
String key = null;
for (Map.Entry<String, Region> entry: rs.onlineRegions.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 3674303..2095dcd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.spy;
@@ -47,7 +46,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -74,6 +72,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -105,12 +106,6 @@ import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.mockito.Mockito;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
* Test class for the Store
*/
@@ -371,7 +366,7 @@ public class TestStore {
// There will be no compaction due to threshold above. Last file will not be replaced.
for (int i = 1; i <= storeFileNum - 1; i++) {
// verify the expired store file.
- assertNull(this.store.requestCompaction());
+ assertFalse(this.store.requestCompaction().isPresent());
Collection<StoreFile> sfs = this.store.getStorefiles();
// Ensure i files are gone.
if (minVersions == 0) {
@@ -386,7 +381,7 @@ public class TestStore {
// Let the next store file expired.
edge.incrementTime(sleepTime);
}
- assertNull(this.store.requestCompaction());
+ assertFalse(this.store.requestCompaction().isPresent());
Collection<StoreFile> sfs = this.store.getStorefiles();
// Assert the last expired file is not removed.
@@ -422,7 +417,7 @@ public class TestStore {
Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
// after compact; check the lowest time stamp
- store.compact(store.requestCompaction(), NoLimitThroughputController.INSTANCE, null);
+ store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null);
lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index 54073bc..d25829d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -823,12 +823,12 @@ public abstract class AbstractTestWALReplay {
final HRegion region =
new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
@Override
- protected FlushResult internalFlushcache(final WAL wal, final long myseqid,
- final Collection<Store> storesToFlush, MonitoredTask status,
+ protected FlushResultImpl internalFlushcache(final WAL wal, final long myseqid,
+ final Collection<HStore> storesToFlush, MonitoredTask status,
boolean writeFlushWalMarker)
throws IOException {
LOG.info("InternalFlushCache Invoked");
- FlushResult fs = super.internalFlushcache(wal, myseqid, storesToFlush,
+ FlushResultImpl fs = super.internalFlushcache(wal, myseqid, storesToFlush,
Mockito.mock(MonitoredTask.class), writeFlushWalMarker);
flushcount.incrementAndGet();
return fs;
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
index 656a0c7..d4d22b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.regionserver.ScanInfo;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -263,7 +264,7 @@ public class TestCoprocessorScanPolicy {
public InternalScanner preCompactScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c, Store store,
List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
- InternalScanner s, CompactionRequest request, long readPoint) throws IOException {
+ InternalScanner s,CompactionLifeCycleTracker tracker, long readPoint) throws IOException {
Long newTtl = ttls.get(store.getTableName());
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();