You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2016/08/26 19:37:07 UTC
[1/9] accumulo git commit: ACCUMULO-1055: created
TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest
Repository: accumulo
Updated Branches:
refs/heads/1.8 b08c4b911 -> a8881acfb
refs/heads/master ee1062f2b -> 90e2db545
ACCUMULO-1055: created TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3ca2fde1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3ca2fde1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3ca2fde1
Branch: refs/heads/1.8
Commit: 3ca2fde1feaeff9b1a83fd3505fd4d90b133a26e
Parents: 271800c
Author: milleruntime <mi...@gmail.com>
Authored: Thu Aug 18 08:19:21 2016 -0400
Committer: milleruntime <mi...@gmail.com>
Committed: Tue Aug 23 12:05:59 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../tserver/tablet/DatafileManager.java | 8 +-
.../apache/accumulo/tserver/tablet/Tablet.java | 3 +-
.../tserver/tablet/DatafileManagerTest.java | 111 +++++++++++++++++++
4 files changed, 120 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 5f4f715..9cb331a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,6 +444,8 @@ public enum Property {
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle " + "tablet will be compacted."),
+ TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
+ "The max file size used for a merging minor compaction."),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
+ "Once this limit is reached, the buffered data is sent to the client."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index b488e13..cef1482 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -58,15 +58,17 @@ class DatafileManager {
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
private final Tablet tablet;
+ private final Long maxMergingMinorCompactionFileSize;
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
+ DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long maxMergCompFileSize) {
for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
+ this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
}
private FileRef mergingMinorCompactionFile = null;
@@ -296,11 +298,11 @@ class DatafileManager {
if (datafileSizes.size() >= maxFiles) {
// find the smallest file
- long min = Long.MAX_VALUE;
+ long min = maxMergingMinorCompactionFileSize;
FileRef minName = null;
for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
- if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
+ if (entry.getValue().getSize() <= min && !majorCompactingFiles.contains(entry.getKey())) {
min = entry.getValue().getSize();
minName = entry.getKey();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 6637521..4377b64 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,7 +484,8 @@ public class Tablet implements TabletCommitter {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- datafileManager = new DatafileManager(this, datafiles);
+ Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
+ datafileManager = new DatafileManager(this, datafiles, maxFileSize);
computeNumEntries();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
new file mode 100644
index 0000000..0e9dee9
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for org.apache.accumulo.tserver.tablet.DatafileManager
+ */
+public class DatafileManagerTest {
+ private Tablet tablet;
+ private KeyExtent extent;
+ private TableConfiguration tableConf;
+
+ private SortedMap<FileRef,DataFileValue> createFileMap(String... sa) {
+ SortedMap<FileRef,DataFileValue> ret = new TreeMap<>();
+ for (int i = 0; i < sa.length; i += 2) {
+ ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i + 1]), 1));
+ }
+ return ret;
+ }
+
+ @Before
+ public void setupMockClasses() {
+ tablet = EasyMock.createMock(Tablet.class);
+ extent = EasyMock.createMock(KeyExtent.class);
+ tableConf = EasyMock.createMock(TableConfiguration.class);
+
+ EasyMock.expect(tablet.getExtent()).andReturn(extent);
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
+
+ EasyMock.replay(tablet, tableConf);
+ }
+
+ /*
+ * Test max file size (table.compaction.minor.merge.file.size.max) exceeded when calling reserveMergingMinorCompactionFile
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals(null, mergeFile);
+ }
+
+ /*
+ * Test max files not reached (table.file.max) when calling reserveMergingMinorCompactionFile
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals(null, mergeFile);
+ }
+
+ /*
+ * Test the smallest file is chosen for merging minor compaction
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals("smallfile", mergeFile.path().getName());
+ }
+
+}
[8/9] accumulo git commit: Merge branch 'pr-141' into 1.8
Posted by ct...@apache.org.
Merge branch 'pr-141' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a8881acf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a8881acf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a8881acf
Branch: refs/heads/1.8
Commit: a8881acfbbbc5c085d54d1c7e41e7702111b12ac
Parents: b08c4b9 ff9a603
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Aug 26 14:28:39 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Aug 26 14:28:39 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../tserver/tablet/DatafileManager.java | 11 +-
.../tserver/tablet/DatafileManagerTest.java | 139 +++++++++++++++++++
3 files changed, 150 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[6/9] accumulo git commit: ACCUMULO-1055: changes for DFM to read
TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Posted by ct...@apache.org.
ACCUMULO-1055: changes for DFM to read TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff9a6035
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff9a6035
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff9a6035
Branch: refs/heads/master
Commit: ff9a603526aff49eb5d860fea1e263432b8636dc
Parents: 792277e
Author: Mike Miller <mi...@gmail.com>
Authored: Thu Aug 25 15:39:53 2016 -0400
Committer: Mike Miller <mi...@gmail.com>
Committed: Thu Aug 25 15:39:53 2016 -0400
----------------------------------------------------------------------
.../tserver/tablet/DatafileManager.java | 7 +++--
.../apache/accumulo/tserver/tablet/Tablet.java | 3 +-
.../tserver/tablet/DatafileManagerTest.java | 32 ++++++++++++++------
3 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 60ad194..ac5243a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -31,6 +31,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -58,17 +59,16 @@ class DatafileManager {
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
private final Tablet tablet;
- private final Long maxMergingMinorCompactionFileSize;
+ private Long maxMergingMinorCompactionFileSize;
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long maxMergCompFileSize) {
+ DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
- this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
}
private FileRef mergingMinorCompactionFile = null;
@@ -299,6 +299,7 @@ class DatafileManager {
// find the smallest file
long maxFileSize = Long.MAX_VALUE;
+ maxMergingMinorCompactionFileSize = AccumuloConfiguration.getMemoryInBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
if (maxMergingMinorCompactionFileSize > 0) {
maxFileSize = maxMergingMinorCompactionFileSize;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 4377b64..6637521 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,8 +484,7 @@ public class Tablet implements TabletCommitter {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
- datafileManager = new DatafileManager(this, datafiles, maxFileSize);
+ datafileManager = new DatafileManager(this, datafiles);
computeNumEntries();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index bb1d6e8..990d24d 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
@@ -56,8 +57,6 @@ public class DatafileManagerTest {
EasyMock.expect(tablet.getExtent()).andReturn(extent);
EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
-
- EasyMock.replay(tablet, tableConf);
}
/*
@@ -65,10 +64,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -81,10 +84,11 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -97,10 +101,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -113,10 +121,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+ String maxMergeFileSize = "0";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
[2/9] accumulo git commit: ACCUMULO-1055: created
TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest
Posted by ct...@apache.org.
ACCUMULO-1055: created TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3ca2fde1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3ca2fde1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3ca2fde1
Branch: refs/heads/master
Commit: 3ca2fde1feaeff9b1a83fd3505fd4d90b133a26e
Parents: 271800c
Author: milleruntime <mi...@gmail.com>
Authored: Thu Aug 18 08:19:21 2016 -0400
Committer: milleruntime <mi...@gmail.com>
Committed: Tue Aug 23 12:05:59 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../tserver/tablet/DatafileManager.java | 8 +-
.../apache/accumulo/tserver/tablet/Tablet.java | 3 +-
.../tserver/tablet/DatafileManagerTest.java | 111 +++++++++++++++++++
4 files changed, 120 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 5f4f715..9cb331a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,6 +444,8 @@ public enum Property {
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle " + "tablet will be compacted."),
+ TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
+ "The max file size used for a merging minor compaction."),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
+ "Once this limit is reached, the buffered data is sent to the client."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index b488e13..cef1482 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -58,15 +58,17 @@ class DatafileManager {
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
private final Tablet tablet;
+ private final Long maxMergingMinorCompactionFileSize;
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
+ DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long maxMergCompFileSize) {
for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
+ this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
}
private FileRef mergingMinorCompactionFile = null;
@@ -296,11 +298,11 @@ class DatafileManager {
if (datafileSizes.size() >= maxFiles) {
// find the smallest file
- long min = Long.MAX_VALUE;
+ long min = maxMergingMinorCompactionFileSize;
FileRef minName = null;
for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
- if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
+ if (entry.getValue().getSize() <= min && !majorCompactingFiles.contains(entry.getKey())) {
min = entry.getValue().getSize();
minName = entry.getKey();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 6637521..4377b64 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,7 +484,8 @@ public class Tablet implements TabletCommitter {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- datafileManager = new DatafileManager(this, datafiles);
+ Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
+ datafileManager = new DatafileManager(this, datafiles, maxFileSize);
computeNumEntries();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
new file mode 100644
index 0000000..0e9dee9
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for org.apache.accumulo.tserver.tablet.DatafileManager
+ */
+public class DatafileManagerTest {
+ private Tablet tablet;
+ private KeyExtent extent;
+ private TableConfiguration tableConf;
+
+ private SortedMap<FileRef,DataFileValue> createFileMap(String... sa) {
+ SortedMap<FileRef,DataFileValue> ret = new TreeMap<>();
+ for (int i = 0; i < sa.length; i += 2) {
+ ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i + 1]), 1));
+ }
+ return ret;
+ }
+
+ @Before
+ public void setupMockClasses() {
+ tablet = EasyMock.createMock(Tablet.class);
+ extent = EasyMock.createMock(KeyExtent.class);
+ tableConf = EasyMock.createMock(TableConfiguration.class);
+
+ EasyMock.expect(tablet.getExtent()).andReturn(extent);
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
+
+ EasyMock.replay(tablet, tableConf);
+ }
+
+ /*
+ * Test max file size (table.compaction.minor.merge.file.size.max) exceeded when calling reserveMergingMinorCompactionFile
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals(null, mergeFile);
+ }
+
+ /*
+ * Test max files not reached (table.file.max) when calling reserveMergingMinorCompactionFile
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals(null, mergeFile);
+ }
+
+ /*
+ * Test the smallest file is chosen for merging minor compaction
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFile() throws IOException {
+ Long maxMergeFileSize = Long.valueOf(1000);
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals("smallfile", mergeFile.path().getName());
+ }
+
+}
[7/9] accumulo git commit: Merge branch 'pr-141' into 1.8
Posted by ct...@apache.org.
Merge branch 'pr-141' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a8881acf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a8881acf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a8881acf
Branch: refs/heads/master
Commit: a8881acfbbbc5c085d54d1c7e41e7702111b12ac
Parents: b08c4b9 ff9a603
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Aug 26 14:28:39 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Aug 26 14:28:39 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../tserver/tablet/DatafileManager.java | 11 +-
.../tserver/tablet/DatafileManagerTest.java | 139 +++++++++++++++++++
3 files changed, 150 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[9/9] accumulo git commit: Merge branch '1.8'
Posted by ct...@apache.org.
Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/90e2db54
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/90e2db54
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/90e2db54
Branch: refs/heads/master
Commit: 90e2db545832cb9b75d6651bbc0a593a2f0831c5
Parents: ee1062f a8881ac
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Aug 26 15:36:05 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Aug 26 15:36:05 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../tserver/tablet/DatafileManager.java | 11 +-
.../tserver/tablet/DatafileManagerTest.java | 139 +++++++++++++++++++
3 files changed, 150 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[3/9] accumulo git commit: ACCUMULO-1055: made default for max MINC
merge file size 0
Posted by ct...@apache.org.
ACCUMULO-1055: made default for max MINC merge file size 0
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/792277e3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/792277e3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/792277e3
Branch: refs/heads/1.8
Commit: 792277e361e24bdbc417da1522206352b1a8a61d
Parents: 3ca2fde
Author: milleruntime <mi...@gmail.com>
Authored: Wed Aug 24 13:53:48 2016 -0400
Committer: milleruntime <mi...@gmail.com>
Committed: Wed Aug 24 13:53:48 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 4 ++--
.../tserver/tablet/DatafileManager.java | 6 +++++-
.../tserver/tablet/DatafileManagerTest.java | 22 +++++++++++++++++---
3 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9cb331a..ce725fb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,8 +444,8 @@ public enum Property {
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle " + "tablet will be compacted."),
- TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
- "The max file size used for a merging minor compaction."),
+ TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "0", PropertyType.MEMORY,
+ "The max file size used for a merging minor compaction. The default value of 0 disables a max file size."),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
+ "Once this limit is reached, the buffered data is sent to the client."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index cef1482..60ad194 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -298,7 +298,11 @@ class DatafileManager {
if (datafileSizes.size() >= maxFiles) {
// find the smallest file
- long min = maxMergingMinorCompactionFileSize;
+ long maxFileSize = Long.MAX_VALUE;
+ if (maxMergingMinorCompactionFileSize > 0) {
+ maxFileSize = maxMergingMinorCompactionFileSize;
+ }
+ long min = maxFileSize;
FileRef minName = null;
for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index 0e9dee9..bb1d6e8 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -65,7 +65,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -81,7 +81,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -97,7 +97,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -108,4 +108,20 @@ public class DatafileManagerTest {
assertEquals("smallfile", mergeFile.path().getName());
}
+ /*
+ * Test disabled max file size for merging minor compaction
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals("smallishfile", mergeFile.path().getName());
+ }
+
}
[5/9] accumulo git commit: ACCUMULO-1055: changes for DFM to read
TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Posted by ct...@apache.org.
ACCUMULO-1055: changes for DFM to read TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff9a6035
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff9a6035
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff9a6035
Branch: refs/heads/1.8
Commit: ff9a603526aff49eb5d860fea1e263432b8636dc
Parents: 792277e
Author: Mike Miller <mi...@gmail.com>
Authored: Thu Aug 25 15:39:53 2016 -0400
Committer: Mike Miller <mi...@gmail.com>
Committed: Thu Aug 25 15:39:53 2016 -0400
----------------------------------------------------------------------
.../tserver/tablet/DatafileManager.java | 7 +++--
.../apache/accumulo/tserver/tablet/Tablet.java | 3 +-
.../tserver/tablet/DatafileManagerTest.java | 32 ++++++++++++++------
3 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 60ad194..ac5243a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -31,6 +31,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -58,17 +59,16 @@ class DatafileManager {
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
private final Tablet tablet;
- private final Long maxMergingMinorCompactionFileSize;
+ private Long maxMergingMinorCompactionFileSize;
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long maxMergCompFileSize) {
+ DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
- this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
}
private FileRef mergingMinorCompactionFile = null;
@@ -299,6 +299,7 @@ class DatafileManager {
// find the smallest file
long maxFileSize = Long.MAX_VALUE;
+ maxMergingMinorCompactionFileSize = AccumuloConfiguration.getMemoryInBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
if (maxMergingMinorCompactionFileSize > 0) {
maxFileSize = maxMergingMinorCompactionFileSize;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 4377b64..6637521 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,8 +484,7 @@ public class Tablet implements TabletCommitter {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
- datafileManager = new DatafileManager(this, datafiles, maxFileSize);
+ datafileManager = new DatafileManager(this, datafiles);
computeNumEntries();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index bb1d6e8..990d24d 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
@@ -56,8 +57,6 @@ public class DatafileManagerTest {
EasyMock.expect(tablet.getExtent()).andReturn(extent);
EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
-
- EasyMock.replay(tablet, tableConf);
}
/*
@@ -65,10 +64,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -81,10 +84,11 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -97,10 +101,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -113,10 +121,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+ String maxMergeFileSize = "0";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
[4/9] accumulo git commit: ACCUMULO-1055: made default for max MINC
merge file size 0
Posted by ct...@apache.org.
ACCUMULO-1055: made default for max MINC merge file size 0
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/792277e3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/792277e3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/792277e3
Branch: refs/heads/master
Commit: 792277e361e24bdbc417da1522206352b1a8a61d
Parents: 3ca2fde
Author: milleruntime <mi...@gmail.com>
Authored: Wed Aug 24 13:53:48 2016 -0400
Committer: milleruntime <mi...@gmail.com>
Committed: Wed Aug 24 13:53:48 2016 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 4 ++--
.../tserver/tablet/DatafileManager.java | 6 +++++-
.../tserver/tablet/DatafileManagerTest.java | 22 +++++++++++++++++---
3 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9cb331a..ce725fb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,8 +444,8 @@ public enum Property {
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle " + "tablet will be compacted."),
- TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
- "The max file size used for a merging minor compaction."),
+ TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "0", PropertyType.MEMORY,
+ "The max file size used for a merging minor compaction. The default value of 0 disables a max file size."),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
+ "Once this limit is reached, the buffered data is sent to the client."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index cef1482..60ad194 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -298,7 +298,11 @@ class DatafileManager {
if (datafileSizes.size() >= maxFiles) {
// find the smallest file
- long min = maxMergingMinorCompactionFileSize;
+ long maxFileSize = Long.MAX_VALUE;
+ if (maxMergingMinorCompactionFileSize > 0) {
+ maxFileSize = maxMergingMinorCompactionFileSize;
+ }
+ long min = maxFileSize;
FileRef minName = null;
for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index 0e9dee9..bb1d6e8 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -65,7 +65,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -81,7 +81,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -97,7 +97,7 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile() throws IOException {
- Long maxMergeFileSize = Long.valueOf(1000);
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -108,4 +108,20 @@ public class DatafileManagerTest {
assertEquals("smallfile", mergeFile.path().getName());
}
+ /*
+ * Test disabled max file size for merging minor compaction
+ */
+ @Test
+ public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
+ Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+ SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+ DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+ EasyMock.verify(tablet, tableConf);
+
+ assertEquals("smallishfile", mergeFile.path().getName());
+ }
+
}