You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2016/08/26 19:37:12 UTC
[6/9] accumulo git commit: ACCUMULO-1055: changes for DFM to read
TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
ACCUMULO-1055: changes for DFM to read TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff9a6035
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff9a6035
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff9a6035
Branch: refs/heads/master
Commit: ff9a603526aff49eb5d860fea1e263432b8636dc
Parents: 792277e
Author: Mike Miller <mi...@gmail.com>
Authored: Thu Aug 25 15:39:53 2016 -0400
Committer: Mike Miller <mi...@gmail.com>
Committed: Thu Aug 25 15:39:53 2016 -0400
----------------------------------------------------------------------
.../tserver/tablet/DatafileManager.java | 7 +++--
.../apache/accumulo/tserver/tablet/Tablet.java | 3 +-
.../tserver/tablet/DatafileManagerTest.java | 32 ++++++++++++++------
3 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 60ad194..ac5243a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -31,6 +31,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -58,17 +59,16 @@ class DatafileManager {
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
private final Tablet tablet;
- private final Long maxMergingMinorCompactionFileSize;
+ private Long maxMergingMinorCompactionFileSize;
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long maxMergCompFileSize) {
+ DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
- this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
}
private FileRef mergingMinorCompactionFile = null;
@@ -299,6 +299,7 @@ class DatafileManager {
// find the smallest file
long maxFileSize = Long.MAX_VALUE;
+ maxMergingMinorCompactionFileSize = AccumuloConfiguration.getMemoryInBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
if (maxMergingMinorCompactionFileSize > 0) {
maxFileSize = maxMergingMinorCompactionFileSize;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 4377b64..6637521 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,8 +484,7 @@ public class Tablet implements TabletCommitter {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
- datafileManager = new DatafileManager(this, datafiles, maxFileSize);
+ datafileManager = new DatafileManager(this, datafiles);
computeNumEntries();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index bb1d6e8..990d24d 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
@@ -56,8 +57,6 @@ public class DatafileManagerTest {
EasyMock.expect(tablet.getExtent()).andReturn(extent);
EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
-
- EasyMock.replay(tablet, tableConf);
}
/*
@@ -65,10 +64,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -81,10 +84,11 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -97,10 +101,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFile() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+ String maxMergeFileSize = "1000B";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);
@@ -113,10 +121,14 @@ public class DatafileManagerTest {
*/
@Test
public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
- Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+ String maxMergeFileSize = "0";
+ EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+ EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+ EasyMock.replay(tablet, tableConf);
+
SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
- DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+ DatafileManager dfm = new DatafileManager(tablet, testFiles);
FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
EasyMock.verify(tablet, tableConf);