You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2014/06/30 17:23:41 UTC
[1/2] git commit: ACCUMULO-2952 move tablets to multiple destinations
in one pass
Repository: accumulo
Updated Branches:
refs/heads/master dbb9cc4a2 -> 1b89892da
ACCUMULO-2952 move tablets to multiple destinations in one pass
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1425450e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1425450e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1425450e
Branch: refs/heads/master
Commit: 1425450e91b90895537e98a226628d55ca9b64f9
Parents: dbb9cc4
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 30 11:17:08 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 30 11:20:00 2014 -0400
----------------------------------------------------------------------
.../master/balancer/DefaultLoadBalancer.java | 21 +++--
.../balancer/DefaultLoadBalancerTest.java | 2 +-
.../org/apache/accumulo/test/BalanceFaster.java | 80 ++++++++++++++++++++
3 files changed, 94 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
index 1fcab46..46b9b5f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
@@ -110,6 +110,7 @@ public class DefaultLoadBalancer extends TabletBalancer {
if (current.size() < 2) {
return false;
}
+ final Map<String,Map<KeyExtent,TabletStats>> donerTabletStats = new HashMap<String,Map<KeyExtent,TabletStats>>();
// Sort by total number of online tablets, per server
int total = 0;
@@ -143,7 +144,8 @@ public class DefaultLoadBalancer extends TabletBalancer {
// under-loaded servers.
int end = totals.size() - 1;
int movedAlready = 0;
- for (int tooManyIndex = 0; tooManyIndex < totals.size(); tooManyIndex++) {
+ int tooManyIndex = 0;
+ while (tooManyIndex < totals.size() && end > tooManyIndex) {
ServerCounts tooMany = totals.get(tooManyIndex);
int goal = even;
if (tooManyIndex < numServersOverEven) {
@@ -156,15 +158,19 @@ public class DefaultLoadBalancer extends TabletBalancer {
break;
}
if (needToUnload >= needToLoad) {
- result.addAll(move(tooMany, tooLittle, needToLoad));
+ result.addAll(move(tooMany, tooLittle, needToLoad, donerTabletStats));
end--;
movedAlready = 0;
} else {
- result.addAll(move(tooMany, tooLittle, needToUnload));
+ result.addAll(move(tooMany, tooLittle, needToUnload, donerTabletStats));
movedAlready += needToUnload;
}
- if (needToUnload > needToLoad)
+ if (needToUnload > needToLoad) {
moreBalancingNeeded = true;
+ } else {
+ tooManyIndex++;
+ donerTabletStats.clear();
+ }
}
} finally {
@@ -186,13 +192,12 @@ public class DefaultLoadBalancer extends TabletBalancer {
/**
* Select a tablet based on differences between table loads; if the loads are even, use the busiest table
*/
- List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count) {
+ List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count, Map<String,Map<KeyExtent,TabletStats>> donerTabletStats) {
List<TabletMigration> result = new ArrayList<TabletMigration>();
if (count == 0)
return result;
- Map<String,Map<KeyExtent,TabletStats>> onlineTablets = new HashMap<String,Map<KeyExtent,TabletStats>>();
// Copy counts so we can update them as we propose migrations
Map<String,Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
Map<String,Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
@@ -224,13 +229,13 @@ public class DefaultLoadBalancer extends TabletBalancer {
// just balance the given table
table = tableToBalance;
}
- Map<KeyExtent,TabletStats> onlineTabletsForTable = onlineTablets.get(table);
+ Map<KeyExtent,TabletStats> onlineTabletsForTable = donerTabletStats.get(table);
try {
if (onlineTabletsForTable == null) {
onlineTabletsForTable = new HashMap<KeyExtent,TabletStats>();
for (TabletStats stat : getOnlineTabletsForTable(tooMuch.server, table))
onlineTabletsForTable.put(new KeyExtent(stat.extent), stat);
- onlineTablets.put(table, onlineTabletsForTable);
+ donerTabletStats.put(table, onlineTabletsForTable);
}
} catch (Exception ex) {
log.error("Unable to select a tablet to move", ex);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
index 9f99b1c..0439429 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
@@ -203,7 +203,7 @@ public class DefaultLoadBalancerTest {
servers.get(migration.newServer).extents.add(migration.tablet);
}
}
- assertEquals(8, moved);
+ assertEquals(9, moved);
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java b/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
new file mode 100644
index 0000000..a6fe5d3
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-2952
+public class BalanceFaster extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(3);
+ }
+
+ @Test(timeout=30*1000)
+ public void test() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ Connector conn = getConnector();
+ conn.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 1000; i++) {
+ splits.add(new Text("" + i));
+ }
+ conn.tableOperations().addSplits(tableName, splits);
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ UtilWaitThread.sleep(5000);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ s.setRange(MetadataSchema.TabletsSection.getRange());
+ Map<String, Integer> counts = new HashMap<String, Integer>();
+ for (Entry<Key,Value> kv : s) {
+ String host = kv.getValue().toString();
+ if (!counts.containsKey(host))
+ counts.put(host, 0);
+ counts.put(host, counts.get(host) + 1);
+ }
+ assertTrue(counts.size() == 3);
+ Iterator<Integer> i = counts.values().iterator();
+ int a = i.next();
+ int b = i.next();
+ int c = i.next();
+ assertTrue(Math.abs(a - b) < 3);
+ assertTrue(Math.abs(a - c) < 3);
+ }
+
+}
[2/2] git commit: ACCUMULO-2952 move IT to proper location
Posted by ec...@apache.org.
ACCUMULO-2952 move IT to proper location
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1b89892d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1b89892d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1b89892d
Branch: refs/heads/master
Commit: 1b89892da316edc4ba1d76efb3ca94f8447a6515
Parents: 1425450
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 30 11:22:26 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 30 11:22:26 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/test/BalanceFaster.java | 80 ------------------
.../apache/accumulo/test/BalanceFasterIT.java | 86 ++++++++++++++++++++
2 files changed, 86 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b89892d/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java b/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
deleted file mode 100644
index a6fe5d3..0000000
--- a/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.junit.Assert.*;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-// ACCUMULO-2952
-public class BalanceFaster extends ConfigurableMacIT {
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setNumTservers(3);
- }
-
- @Test(timeout=30*1000)
- public void test() throws Exception {
- String tableName = getUniqueNames(1)[0];
- Connector conn = getConnector();
- conn.tableOperations().create(tableName);
- SortedSet<Text> splits = new TreeSet<Text>();
- for (int i = 0; i < 1000; i++) {
- splits.add(new Text("" + i));
- }
- conn.tableOperations().addSplits(tableName, splits);
- Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- UtilWaitThread.sleep(5000);
- s.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
- s.setRange(MetadataSchema.TabletsSection.getRange());
- Map<String, Integer> counts = new HashMap<String, Integer>();
- for (Entry<Key,Value> kv : s) {
- String host = kv.getValue().toString();
- if (!counts.containsKey(host))
- counts.put(host, 0);
- counts.put(host, counts.get(host) + 1);
- }
- assertTrue(counts.size() == 3);
- Iterator<Integer> i = counts.values().iterator();
- int a = i.next();
- int b = i.next();
- int c = i.next();
- assertTrue(Math.abs(a - b) < 3);
- assertTrue(Math.abs(a - c) < 3);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b89892d/test/src/test/java/org/apache/accumulo/test/BalanceFasterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/BalanceFasterIT.java b/test/src/test/java/org/apache/accumulo/test/BalanceFasterIT.java
new file mode 100644
index 0000000..a1a7f1c
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/BalanceFasterIT.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-2952
+public class BalanceFasterIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(3);
+ }
+
+ @Test(timeout=30*1000)
+ public void test() throws Exception {
+ // create a table, add a bunch of splits
+ String tableName = getUniqueNames(1)[0];
+ Connector conn = getConnector();
+ conn.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 1000; i++) {
+ splits.add(new Text("" + i));
+ }
+ conn.tableOperations().addSplits(tableName, splits);
+ // give a short wait for balancing
+ UtilWaitThread.sleep(10*1000);
+ // find out where the tabets are
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ s.setRange(MetadataSchema.TabletsSection.getRange());
+ Map<String, Integer> counts = new HashMap<String, Integer>();
+ for (Entry<Key,Value> kv : s) {
+ String host = kv.getValue().toString();
+ if (!counts.containsKey(host))
+ counts.put(host, 0);
+ counts.put(host, counts.get(host) + 1);
+ }
+ // should be on all three servers
+ assertTrue(counts.size() == 3);
+ // and distributed evenly
+ Iterator<Integer> i = counts.values().iterator();
+ int a = i.next();
+ int b = i.next();
+ int c = i.next();
+ assertTrue(Math.abs(a - b) < 3);
+ assertTrue(Math.abs(a - c) < 3);
+ assertTrue(a > 330);
+ }
+
+}