You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/09/10 05:40:58 UTC
git commit: ACCUMULO-1689 Add a simple test to make sure we get rid
of some files after compacting tables which we wrote data to.
Updated Branches:
refs/heads/1.4.5-SNAPSHOT f868c3762 -> 76c3c8490
ACCUMULO-1689 Add a simple test to make sure we get rid of some files after
compacting tables which we wrote data to.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76c3c849
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76c3c849
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76c3c849
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 76c3c8490df6e5a81f40d5db7c084fd4d5d716b2
Parents: f868c37
Author: Josh Elser <el...@apache.org>
Authored: Mon Sep 9 23:41:01 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Sep 9 23:41:01 2013 -0400
----------------------------------------------------------------------
pom.xml | 6 +
src/minicluster/pom.xml | 5 +
.../minicluster/MiniAccumuloClusterGCTest.java | 119 +++++++++++++++++++
3 files changed, 130 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d427da8..9ed2fdf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -651,6 +651,12 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>14.0.1</version>
+ <scope>test</scope>
+ </dependency>
<!-- provided dependencies needed at runtime -->
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/src/minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/src/minicluster/pom.xml b/src/minicluster/pom.xml
index bb85148..3e23b0c 100644
--- a/src/minicluster/pom.xml
+++ b/src/minicluster/pom.xml
@@ -104,6 +104,11 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
new file mode 100644
index 0000000..7c57e7e
--- /dev/null
+++ b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ *
+ */
+public class MiniAccumuloClusterGCTest {
+
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloConfig macConfig;
+ private static MiniAccumuloCluster accumulo;
+ private static final String passwd = "password";
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+ tmpDir.create();
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ macConfig = new MiniAccumuloConfig(tmpDir.getRoot(), passwd);
+ macConfig.setNumTservers(1);
+
+ // Turn on the garbage collector
+ macConfig.runGC(true);
+
+ // And tweak the settings to make it run often
+ Map<String,String> config = ImmutableMap.of(Property.GC_CYCLE_DELAY.getKey(), "1s", Property.GC_CYCLE_START.getKey(), "0s");
+ macConfig.setSiteConfig(config);
+
+ accumulo = new MiniAccumuloCluster(macConfig);
+ accumulo.start();
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+
+ @Test(timeout = 20000)
+ public void test() throws Exception {
+ ZooKeeperInstance inst = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+ Connector c = inst.getConnector("root", passwd);
+
+ final String table = "foobar";
+ c.tableOperations().create(table);
+
+ BatchWriter bw = null;
+
+ // Add some data
+ try {
+ bw = c.createBatchWriter(table, 1000l, 100l, 1);
+ Mutation m = new Mutation("a");
+ for (int i = 0; i < 50; i++) {
+ m.put("colf", Integer.toString(i), "");
+ }
+
+ bw.addMutation(m);
+ } finally {
+ if (null != bw) {
+ bw.close();
+ }
+ }
+
+ final boolean flush = true, wait = true;
+
+ // Compact the tables to get some rfiles which we can gc
+ c.tableOperations().compact(table, null, null, flush, wait);
+ c.tableOperations().compact("!METADATA", null, null, flush, wait);
+
+ File accumuloDir = new File(tmpDir.getRoot().getAbsolutePath(), "accumulo");
+ File tables = new File(accumuloDir.getAbsolutePath(), "tables");
+
+ int fileCountAfterCompaction = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE).size();
+
+ // Sleep for 4s to let the GC do its thing
+ Thread.sleep(4000);
+
+ int fileCountAfterGCWait = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE).size();
+
+ Assert.assertTrue("Expected to find less files after compaction and pause for GC", fileCountAfterGCWait < fileCountAfterCompaction);
+ }
+
+}