You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/03/07 10:14:47 UTC
[11/33] hbase git commit: HBASE-18135 Implement mechanism for
RegionServers to report file archival for space quotas
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java
new file mode 100644
index 0000000..e139e4f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
+
+/**
+ * Test class for {@link FileArchiverNotifierImpl}.
+ */
+@Category(MediumTests.class)
+public class TestFileArchiverNotifierImpl {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong();
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private Connection conn;
+ private Admin admin;
+ private SpaceQuotaHelperForTests helper;
+ private FileSystem fs;
+ private Configuration conf;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ // Clean up the compacted files faster than normal (15s instead of 2mins)
+ conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
+ // Prevent the SnapshotQuotaObserverChore from running
+ conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
+ conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ conn = TEST_UTIL.getConnection();
+ admin = TEST_UTIL.getAdmin();
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
+ helper.removeAllQuotas(conn);
+ fs = TEST_UTIL.getTestFileSystem();
+ conf = TEST_UTIL.getConfiguration();
+ }
+
+ @Test
+ public void testSnapshotSizePersistence() throws IOException {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final TableName tn = TableName.valueOf(testName.getMethodName());
+ if (admin.tableExists(tn)) {
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ }
+ TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn).addColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
+ admin.createTable(desc);
+
+ FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
+ List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
+ try (Table table = conn.getTable(tn)) {
+ // Writing no values will result in no records written.
+ verify(table, () -> {
+ notifier.persistSnapshotSizes(table, snapshotsWithSizes);
+ assertEquals(0, count(table));
+ });
+
+ verify(table, () -> {
+ snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
+ snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
+ notifier.persistSnapshotSizes(table, snapshotsWithSizes);
+ assertEquals(2, count(table));
+ assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
+ assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
+ });
+ }
+ }
+
+ @Test
+ public void testIncrementalFileArchiving() throws Exception {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final TableName tn = TableName.valueOf(testName.getMethodName());
+ if (admin.tableExists(tn)) {
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ }
+ final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
+ final TableName tn1 = helper.createTableWithRegions(1);
+ admin.setQuota(QuotaSettingsFactory.limitTableSpace(
+ tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
+
+ // Write some data and flush it
+ helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
+ admin.flush(tn1);
+
+ // Create a snapshot on the table
+ final String snapshotName1 = tn1 + "snapshot1";
+ admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
+
+ FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
+ long t1 = notifier.getLastFullCompute();
+ long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
+ assertEquals("The size of the snapshots should be zero", 0, snapshotSize);
+ assertTrue("Last compute time was not less than current compute time",
+ t1 < notifier.getLastFullCompute());
+
+ // No recently archived files and the snapshot should have no size
+ assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
+
+ // Invoke the addArchivedFiles method with no files
+ notifier.addArchivedFiles(Collections.emptySet());
+
+ // The size should not have changed
+ assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
+
+ notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
+
+ // The size should not have changed
+ assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
+
+ // Pull one file referenced by the snapshot out of the manifest
+ Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
+ assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1);
+ String referencedFile = Iterables.getFirst(referencedFiles, null);
+ assertNotNull(referencedFile);
+
+ // Report that a file this snapshot referenced was moved to the archive. This is a sign
+ // that the snapshot should now "own" the size of this file
+ final long fakeFileSize = 2048L;
+ notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
+
+ // Verify that the snapshot owns this file.
+ assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
+
+ // In reality, we did not actually move the file, so a "full" computation should re-set the
+ // size of the snapshot back to 0.
+ long t2 = notifier.getLastFullCompute();
+ snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
+ assertEquals(0, snapshotSize);
+ assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
+ // We should also have no recently archived files after a re-computation
+ assertTrue("Last compute time was not less than current compute time",
+ t2 < notifier.getLastFullCompute());
+ }
+
+ @Test
+ public void testParseOldNamespaceSnapshotSize() throws Exception {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName());
+ final TableName tn = TableName.valueOf(testName.getMethodName() + "1");
+ if (admin.tableExists(fakeQuotaTableName)) {
+ admin.disableTable(fakeQuotaTableName);
+ admin.deleteTable(fakeQuotaTableName);
+ }
+ TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName).addColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
+ admin.createTable(desc);
+
+ final String ns = "";
+ try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
+ FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
+ // Verify no record is treated as zero
+ assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
+
+ // Set an explicit value of zero
+ fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
+ assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
+
+ // Set a non-zero value
+ fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
+ assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
+ }
+ }
+
+ private long count(Table t) throws IOException {
+ try (ResultScanner rs = t.getScanner(new Scan())) {
+ long sum = 0;
+ for (Result r : rs) {
+ while (r.advance()) {
+ sum++;
+ }
+ }
+ return sum;
+ }
+ }
+
+ private long extractSnapshotSize(
+ Table quotaTable, TableName tn, String snapshot) throws IOException {
+ Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
+ Result r = quotaTable.get(g);
+ assertNotNull(r);
+ CellScanner cs = r.cellScanner();
+ assertTrue(cs.advance());
+ Cell c = cs.current();
+ assertNotNull(c);
+ return QuotaTableUtil.extractSnapshotSize(
+ c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ }
+
+ private void verify(Table t, IOThrowingRunnable test) throws IOException {
+ admin.disableTable(t.getName());
+ admin.truncateTable(t.getName(), false);
+ test.run();
+ }
+
+ @FunctionalInterface
+ private interface IOThrowingRunnable {
+ void run() throws IOException;
+ }
+
+ private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
+ HashSet<String> files = new HashSet<>();
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+ snapshotName, FSUtils.getRootDir(conf));
+ SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
+ fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
+ // For each region referenced by the snapshot
+ for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
+ // For each column family in this region
+ for (FamilyFiles ff : rm.getFamilyFilesList()) {
+ // And each store file in that family
+ for (StoreFile sf : ff.getStoreFilesList()) {
+ files.add(sf.getName());
+ }
+ }
+ }
+ return files;
+ }
+
+ private <K,V> Entry<K,V> entry(K k, V v) {
+ return Maps.immutableEntry(k, v);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
index f90ed82..fdc7ad3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.quotas;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.Collections;
+import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
@@ -31,10 +33,15 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClientServiceCallable;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RpcRetryingCaller;
import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
+import org.apache.hadoop.hbase.client.SnapshotType;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.AfterClass;
import org.junit.Before;
@@ -69,8 +76,15 @@ public class TestLowLatencySpaceQuotas {
Configuration conf = TEST_UTIL.getConfiguration();
// The default 1s period for QuotaObserverChore is good.
SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
- // Set the period to read region size from HDFS to be very long
+ // Set the period/delay to read region size from HDFS to be very long
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000 * 120);
+ conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000 * 120);
+ // Set the same long period/delay to compute snapshot sizes
+ conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 1000 * 120);
+ conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 1000 * 120);
+ // Clean up the compacted files faster than normal (5s instead of 2mins)
+ conf.setInt("hbase.hfile.compaction.discharger.interval", 5 * 1000);
+
TEST_UTIL.startMiniCluster(1);
}
@@ -225,4 +239,69 @@ public class TestLowLatencySpaceQuotas {
}
});
}
+
+ @Test
+ public void testSnapshotSizes() throws Exception {
+ TableName tn = helper.createTableWithRegions(1);
+ // Set a quota
+ QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
+ tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
+ admin.setQuota(settings);
+
+ // Write some data and flush it to disk.
+ final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+ helper.writeData(tn, sizePerBatch);
+ admin.flush(tn);
+
+ final String snapshot1 = "snapshot1";
+ admin.snapshot(snapshot1, tn, SnapshotType.SKIPFLUSH);
+
+ // Compute the size of the file for the Region we'll send to archive
+ Region region = Iterables.getOnlyElement(TEST_UTIL.getHBaseCluster().getRegions(tn));
+ List<? extends Store> stores = region.getStores();
+ long summer = 0;
+ for (Store store : stores) {
+ summer += store.getStorefilesSize();
+ }
+ final long storeFileSize = summer;
+
+ // Wait for the table to show the usage
+ TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
+ @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
+ return snapshot.getUsage() == storeFileSize;
+ }
+ });
+
+ // Spoof a "full" computation of snapshot size. Normally the chore handles this, but we want
+ // to test in the absence of this chore.
+ FileArchiverNotifier notifier = TEST_UTIL.getHBaseCluster().getMaster()
+ .getSnapshotQuotaObserverChore().getNotifierForTable(tn);
+ notifier.computeAndStoreSnapshotSizes(Collections.singletonList(snapshot1));
+
+ // Force a major compaction to create a new file and push the old file to the archive
+ TEST_UTIL.compact(tn, true);
+
+ // After moving the old file to archive/, the space of this table should double
+ // We have a new file created by the majc referenced by the table and the snapshot still
+ // referencing the old file.
+ TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
+ @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
+ return snapshot.getUsage() >= 2 * storeFileSize;
+ }
+ });
+
+ try (Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
+ Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot1));
+ assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty());
+ assertTrue(r.advance());
+ assertEquals("The snapshot's size should be the same as the origin store file",
+ storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current()));
+
+ r = quotaTable.get(QuotaTableUtil.createGetNamespaceSnapshotSize(tn.getNamespaceAsString()));
+ assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty());
+ assertTrue(r.advance());
+ assertEquals("The snapshot's size should be the same as the origin store file",
+ storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current()));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a4c0120/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java
index 818d6ff..097e646 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java
@@ -18,33 +18,35 @@
package org.apache.hadoop.hbase.quotas;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.SnapshotWithSize;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
import org.apache.hadoop.hbase.regionserver.HStore;
@@ -61,7 +63,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
-import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
/**
@@ -114,50 +116,6 @@ public class TestSnapshotQuotaObserverChore {
}
@Test
- public void testSnapshotSizePersistence() throws IOException {
- final Admin admin = TEST_UTIL.getAdmin();
- final TableName tn = TableName.valueOf("quota_snapshotSizePersistence");
- if (admin.tableExists(tn)) {
- admin.disableTable(tn);
- admin.deleteTable(tn);
- }
- HTableDescriptor desc = new HTableDescriptor(tn);
- desc.addFamily(new HColumnDescriptor(QuotaTableUtil.QUOTA_FAMILY_USAGE));
- admin.createTable(desc);
-
- Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
- try (Table table = conn.getTable(tn)) {
- // Writing no values will result in no records written.
- verify(table, () -> {
- testChore.persistSnapshotSizes(table, snapshotsWithSizes);
- assertEquals(0, count(table));
- });
-
- verify(table, () -> {
- TableName originatingTable = TableName.valueOf("t1");
- snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss1", 1024L));
- snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss2", 4096L));
- testChore.persistSnapshotSizes(table, snapshotsWithSizes);
- assertEquals(2, count(table));
- assertEquals(1024L, extractSnapshotSize(table, originatingTable, "ss1"));
- assertEquals(4096L, extractSnapshotSize(table, originatingTable, "ss2"));
- });
-
- snapshotsWithSizes.clear();
- verify(table, () -> {
- snapshotsWithSizes.put(TableName.valueOf("t1"), new SnapshotWithSize("ss1", 1024L));
- snapshotsWithSizes.put(TableName.valueOf("t2"), new SnapshotWithSize("ss2", 4096L));
- snapshotsWithSizes.put(TableName.valueOf("t3"), new SnapshotWithSize("ss3", 8192L));
- testChore.persistSnapshotSizes(table, snapshotsWithSizes);
- assertEquals(3, count(table));
- assertEquals(1024L, extractSnapshotSize(table, TableName.valueOf("t1"), "ss1"));
- assertEquals(4096L, extractSnapshotSize(table, TableName.valueOf("t2"), "ss2"));
- assertEquals(8192L, extractSnapshotSize(table, TableName.valueOf("t3"), "ss3"));
- });
- }
- }
-
- @Test
public void testSnapshotsFromTables() throws Exception {
TableName tn1 = helper.createTableWithRegions(1);
TableName tn2 = helper.createTableWithRegions(1);
@@ -268,13 +226,13 @@ public class TestSnapshotQuotaObserverChore {
"Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
// Get the size of our snapshot
- Multimap<TableName,SnapshotWithSize> snapshotsWithSize = testChore.computeSnapshotSizes(
+ Map<String,Long> namespaceSnapshotSizes = testChore.computeSnapshotSizes(
snapshotsToCompute);
- assertEquals(1, snapshotsWithSize.size());
- SnapshotWithSize sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
- assertEquals(snapshotName, sws.getName());
+ assertEquals(1, namespaceSnapshotSizes.size());
+ Long size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
+ assertNotNull(size);
// The snapshot should take up no space since the table refers to it completely
- assertEquals(0, sws.getSize());
+ assertEquals(0, size.longValue());
// Write some more data, flush it, and then major_compact the table
helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
@@ -302,71 +260,132 @@ public class TestSnapshotQuotaObserverChore {
snapshotsToCompute = testChore.getSnapshotsToComputeSize();
assertEquals(
"Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
- snapshotsWithSize = testChore.computeSnapshotSizes(
+ namespaceSnapshotSizes = testChore.computeSnapshotSizes(
snapshotsToCompute);
- assertEquals(1, snapshotsWithSize.size());
- sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
- assertEquals(snapshotName, sws.getName());
+ assertEquals(1, namespaceSnapshotSizes.size());
+ size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
+ assertNotNull(size);
// The snapshot should take up the size the table originally took up
- assertEquals(snapshotSize, sws.getSize());
+ assertEquals(snapshotSize, size.longValue());
}
@Test
public void testPersistingSnapshotsForNamespaces() throws Exception {
- Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
TableName tn1 = TableName.valueOf("ns1:tn1");
TableName tn2 = TableName.valueOf("ns1:tn2");
TableName tn3 = TableName.valueOf("ns2:tn1");
TableName tn4 = TableName.valueOf("ns2:tn2");
TableName tn5 = TableName.valueOf("tn1");
-
- snapshotsWithSizes.put(tn1, new SnapshotWithSize("", 1024L));
- snapshotsWithSizes.put(tn2, new SnapshotWithSize("", 1024L));
- snapshotsWithSizes.put(tn3, new SnapshotWithSize("", 512L));
- snapshotsWithSizes.put(tn4, new SnapshotWithSize("", 1024L));
- snapshotsWithSizes.put(tn5, new SnapshotWithSize("", 3072L));
-
- Map<String,Long> nsSizes = testChore.groupSnapshotSizesByNamespace(snapshotsWithSizes);
- assertEquals(3, nsSizes.size());
- assertEquals(2048L, (long) nsSizes.get("ns1"));
- assertEquals(1536L, (long) nsSizes.get("ns2"));
- assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
+ // Shim in a custom factory to avoid computing snapshot sizes.
+ FileArchiverNotifierFactory test = new FileArchiverNotifierFactory() {
+ Map<TableName,Long> tableToSize = ImmutableMap.of(
+ tn1, 1024L, tn2, 1024L, tn3, 512L, tn4, 1024L, tn5, 3072L);
+ @Override
+ public FileArchiverNotifier get(
+ Connection conn, Configuration conf, FileSystem fs, TableName tn) {
+ return new FileArchiverNotifier() {
+ @Override public void addArchivedFiles(Set<Entry<String,Long>> fileSizes)
+ throws IOException {}
+
+ @Override
+ public long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots)
+ throws IOException {
+ return tableToSize.get(tn);
+ }
+ };
+ }
+ };
+ try {
+ FileArchiverNotifierFactoryImpl.setInstance(test);
+
+ Multimap<TableName,String> snapshotsToCompute = HashMultimap.create();
+ snapshotsToCompute.put(tn1, "");
+ snapshotsToCompute.put(tn2, "");
+ snapshotsToCompute.put(tn3, "");
+ snapshotsToCompute.put(tn4, "");
+ snapshotsToCompute.put(tn5, "");
+ Map<String,Long> nsSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
+ assertEquals(3, nsSizes.size());
+ assertEquals(2048L, (long) nsSizes.get("ns1"));
+ assertEquals(1536L, (long) nsSizes.get("ns2"));
+ assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
+ } finally {
+ FileArchiverNotifierFactoryImpl.reset();
+ }
}
- private long count(Table t) throws IOException {
- try (ResultScanner rs = t.getScanner(new Scan())) {
- long sum = 0;
- for (Result r : rs) {
- while (r.advance()) {
- sum++;
+ @Test
+ public void testBucketingFilesToSnapshots() throws Exception {
+ // Create a table and set a quota
+ TableName tn1 = helper.createTableWithRegions(1);
+ admin.setQuota(QuotaSettingsFactory.limitTableSpace(
+ tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
+
+ // Write some data and flush it
+ helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
+ admin.flush(tn1);
+
+ final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
+ // Wait for the Master chore to run to see the usage (with a fudge factor)
+ TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
+ @Override
+ boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
+ lastSeenSize.set(snapshot.getUsage());
+ return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
+ }
+ });
+
+ // Create a snapshot on the table
+ final String snapshotName1 = tn1 + "snapshot1";
+ admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
+ // Major compact the table to force a rewrite
+ TEST_UTIL.compact(tn1, true);
+
+ // Make sure that the snapshot owns the size
+ final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
+ TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
+ Result r = quotaTable.get(g);
+ if (r == null || r.isEmpty()) {
+ return false;
}
+ r.advance();
+ Cell c = r.current();
+ return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
}
- return sum;
- }
- }
+ });
- private long extractSnapshotSize(
- Table quotaTable, TableName tn, String snapshot) throws IOException {
- Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
- Result r = quotaTable.get(g);
- assertNotNull(r);
- CellScanner cs = r.cellScanner();
- cs.advance();
- Cell c = cs.current();
- assertNotNull(c);
- return QuotaTableUtil.extractSnapshotSize(
- c.getValueArray(), c.getValueOffset(), c.getValueLength());
- }
+ // Create another snapshot on the table
+ final String snapshotName2 = tn1 + "snapshot2";
+ admin.snapshot(new SnapshotDescription(snapshotName2, tn1, SnapshotType.SKIPFLUSH));
+ // Major compact the table to force a rewrite
+ TEST_UTIL.compact(tn1, true);
- private void verify(Table t, IOThrowingRunnable test) throws IOException {
- admin.disableTable(t.getName());
- admin.truncateTable(t.getName(), false);
- test.run();
- }
+ // Make sure that the snapshot owns the size
+ TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName2);
+ Result r = quotaTable.get(g);
+ if (r == null || r.isEmpty()) {
+ return false;
+ }
+ r.advance();
+ Cell c = r.current();
+ return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
+ }
+ });
- @FunctionalInterface
- private interface IOThrowingRunnable {
- void run() throws IOException;
+ Get g = QuotaTableUtil.createGetNamespaceSnapshotSize(tn1.getNamespaceAsString());
+ Result r = quotaTable.get(g);
+ assertNotNull(r);
+ assertFalse(r.isEmpty());
+ r.advance();
+ long size = QuotaTableUtil.parseSnapshotSize(r.current());
+ // Two snapshots of equal size.
+ assertEquals(lastSeenSize.get() * 2, size);
}
/**