[23/49] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[el...@apache.org]

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
new file mode 100644
index 0000000..ec8f1bf
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class BaseViolationPolicyEnforcement {
+
+  static final Append APPEND = new Append(Bytes.toBytes("foo"));
+  static final Delete DELETE = new Delete(Bytes.toBytes("foo"));
+  static final Increment INCREMENT = new Increment(Bytes.toBytes("foo"));
+  static final Put PUT = new Put(Bytes.toBytes("foo"));
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
new file mode 100644
index 0000000..abe1b9d
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestBulkLoadCheckingViolationPolicyEnforcement {
+
+  FileSystem fs;
+  RegionServerServices rss;
+  TableName tableName;
+  SpaceViolationPolicyEnforcement policy;
+
+  @Before
+  public void setup() {
+    fs = mock(FileSystem.class);
+    rss = mock(RegionServerServices.class);
+    tableName = TableName.valueOf("foo");
+    policy = new BulkLoadVerifyingViolationPolicyEnforcement();
+  }
+
+  @Test
+  public void testFilesUnderLimit() throws Exception {
+    final List<String> paths = new ArrayList<>();
+    final List<FileStatus> statuses = new ArrayList<>();
+    final long length = 100L * 1024L;
+    for (int i = 0; i < 5; i++) {
+      String path = "/" + i;
+      FileStatus status = mock(FileStatus.class);
+      when(fs.getFileStatus(new Path(path))).thenReturn(status);
+      when(status.getLen()).thenReturn(length);
+      when(status.isFile()).thenReturn(true);
+      paths.add(path);
+      statuses.add(status);
+    }
+
+    // Quota is not in violation now
+    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6);
+
+    policy.initialize(rss, tableName, snapshot);
+
+    policy.checkBulkLoad(fs, paths);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testFileIsNotAFile() throws Exception {
+    final List<String> paths = new ArrayList<>();
+    String path = "/1";
+    FileStatus status = mock(FileStatus.class);
+    when(fs.getFileStatus(new Path(path))).thenReturn(status);
+    when(status.getLen()).thenReturn(1000L);
+    when(status.isFile()).thenReturn(false);
+    paths.add(path);
+
+    // Quota is not in violation now
+    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE);
+
+    policy.initialize(rss, tableName, snapshot);
+
+    // If the file to bulk load isn't a file, this should throw an exception
+    policy.checkBulkLoad(fs, paths);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testOneFileInBatchOverLimit() throws Exception {
+    final List<String> paths = new ArrayList<>();
+    final List<FileStatus> statuses = new ArrayList<>();
+    final long length = 1000L * 1024L;
+    for (int i = 0; i < 5; i++) {
+      String path = "/" + i;
+      FileStatus status = mock(FileStatus.class);
+      when(fs.getFileStatus(new Path(path))).thenReturn(status);
+      when(status.getLen()).thenReturn(length);
+      when(status.isFile()).thenReturn(true);
+      paths.add(path);
+      statuses.add(status);
+    }
+
+    // Quota is not in violation now
+    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L);
+
+    policy.initialize(rss, tableName, snapshot);
+
+    policy.checkBulkLoad(fs, paths);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testSumOfFilesOverLimit() throws Exception {
+    final List<String> paths = new ArrayList<>();
+    final List<FileStatus> statuses = new ArrayList<>();
+    final long length = 1024L;
+    for (int i = 0; i < 5; i++) {
+      String path = "/" + i;
+      FileStatus status = mock(FileStatus.class);
+      when(fs.getFileStatus(new Path(path))).thenReturn(status);
+      when(status.getLen()).thenReturn(length);
+      when(status.isFile()).thenReturn(true);
+      paths.add(path);
+      statuses.add(status);
+    }
+
+    // Quota is not in violation now, but 5*1024 files would push us to violation
+    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L);
+
+    policy.initialize(rss, tableName, snapshot);
+
+    policy.checkBulkLoad(fs, paths);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java
new file mode 100644
index 0000000..c42d866
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test class for {@link DisableTableViolationPolicyEnforcement}.
+ */
+@Category(SmallTests.class)
+public class TestDisableTableViolationPolicyEnforcement extends BaseViolationPolicyEnforcement {
+
+  private DisableTableViolationPolicyEnforcement enforcement;
+
+  @Before
+  public void setup() {
+    enforcement = new DisableTableViolationPolicyEnforcement();
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckPut() throws SpaceLimitingException {
+    // If the policy is enacted, it will always throw an exception
+    // to avoid constantly re-checking the table state.
+    enforcement.check(PUT);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckAppend() throws SpaceLimitingException {
+    enforcement.check(APPEND);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckDelete() throws SpaceLimitingException {
+    enforcement.check(DELETE);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckIncrement() throws SpaceLimitingException {
+    enforcement.check(INCREMENT);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java
new file mode 100644
index 0000000..1115fab
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test class for {@link NoInsertsViolationPolicyEnforcement}.
+ */
+@Category(SmallTests.class)
+public class TestNoInsertsViolationPolicyEnforcement extends BaseViolationPolicyEnforcement {
+
+  private NoInsertsViolationPolicyEnforcement enforcement;
+
+  @Before
+  public void setup() {
+    enforcement = new NoInsertsViolationPolicyEnforcement();
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckAppend() throws Exception {
+    enforcement.check(APPEND);
+  }
+
+  @Test
+  public void testCheckDelete() throws Exception {
+    enforcement.check(DELETE);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckIncrement() throws Exception {
+    enforcement.check(INCREMENT);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckPut() throws Exception {
+    enforcement.check(PUT);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java
new file mode 100644
index 0000000..1348eb7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test class for {@link NoWritesCompactionsViolationPolicyEnforcement};
+ */
+@Category(SmallTests.class)
+public class TestNoWritesCompactionsViolationPolicyEnforcement
+    extends BaseViolationPolicyEnforcement {
+
+  private NoWritesCompactionsViolationPolicyEnforcement enforcement;
+
+  @Before
+  public void setup() {
+    enforcement = new NoWritesCompactionsViolationPolicyEnforcement();
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckAppend() throws Exception {
+    enforcement.check(APPEND);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckDelete() throws Exception {
+    enforcement.check(DELETE);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckIncrement() throws Exception {
+    enforcement.check(INCREMENT);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckPut() throws Exception {
+    enforcement.check(PUT);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/34ba143f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java
new file mode 100644
index 0000000..c032f26
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test class for {@link NoWritesViolationPolicyEnforcement}.
+ */
+@Category(SmallTests.class)
+public class TestNoWritesViolationPolicyEnforcement extends BaseViolationPolicyEnforcement {
+
+  private NoWritesViolationPolicyEnforcement enforcement;
+
+  @Before
+  public void setup() {
+    enforcement = new NoWritesViolationPolicyEnforcement();
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckAppend() throws Exception {
+    enforcement.check(APPEND);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckDelete() throws Exception {
+    enforcement.check(DELETE);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckIncrement() throws Exception {
+    enforcement.check(INCREMENT);
+  }
+
+  @Test(expected = SpaceLimitingException.class)
+  public void testCheckPut() throws Exception {
+    enforcement.check(PUT);
+  }
+}