You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cs...@apache.org on 2022/12/14 22:24:56 UTC
[accumulo] branch main updated: Remove deprecated TabletBalancer api (#3117)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7df6335ad6 Remove deprecated TabletBalancer api (#3117)
7df6335ad6 is described below
commit 7df6335ad6ebae8e6ff99a19e528ea3b2a8b2af7
Author: Christopher L. Shannon <ch...@gmail.com>
AuthorDate: Wed Dec 14 17:24:52 2022 -0500
Remove deprecated TabletBalancer api (#3117)
Tablet balancer was moved to SPI
Issue #3115
---
.../master/balancer/DefaultLoadBalancer.java | 374 ----------
.../server/master/balancer/GroupBalancer.java | 751 ---------------------
.../balancer/HostRegexTableLoadBalancer.java | 576 ----------------
.../server/master/balancer/RegexGroupBalancer.java | 107 ---
.../server/master/balancer/TableLoadBalancer.java | 166 -----
.../server/master/balancer/TabletBalancer.java | 301 ---------
.../server/master/state/TabletMigration.java | 43 --
.../BaseHostRegexTableLoadBalancerTest.java | 371 ----------
.../master/balancer/DefaultLoadBalancerTest.java | 303 ---------
.../server/master/balancer/GroupBalancerTest.java | 359 ----------
...tRegexTableLoadBalancerReconfigurationTest.java | 132 ----
.../balancer/HostRegexTableLoadBalancerTest.java | 480 -------------
.../master/balancer/TableLoadBalancerTest.java | 192 ------
13 files changed, 4155 deletions(-)
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
deleted file mode 100644
index 9aea12dd6c..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-import java.util.SortedMap;
-
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @deprecated since 2.1.0. Use {@link org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer}
- * instead, as it as the same functionality but a stable API.
- */
-@Deprecated(since = "2.1.0")
-public class DefaultLoadBalancer extends TabletBalancer {
-
- private static final Logger log = LoggerFactory.getLogger(DefaultLoadBalancer.class);
-
- Iterator<TServerInstance> assignments;
- // if tableToBalance is set, then only balance the given table
- TableId tableToBalance = null;
-
- public DefaultLoadBalancer() {
- log.warn(
- "{} has been deprecated and will be removed in a future release. Please update your "
- + "configuration to use the equivalent {} instead.",
- getClass().getName(), SimpleLoadBalancer.class.getName());
- }
-
- public DefaultLoadBalancer(TableId table) {
- this(); // emit warning
- tableToBalance = table;
- }
-
- List<TServerInstance> randomize(Set<TServerInstance> locations) {
- List<TServerInstance> result = new ArrayList<>(locations);
- Collections.shuffle(result);
- return result;
- }
-
- public TServerInstance getAssignment(SortedMap<TServerInstance,TabletServerStatus> locations,
- TServerInstance last) {
- if (locations.isEmpty()) {
- return null;
- }
-
- if (last != null) {
- // Maintain locality
- String fakeSessionID = " ";
- TServerInstance simple = new TServerInstance(last.getHostAndPort(), fakeSessionID);
- Iterator<TServerInstance> find = locations.tailMap(simple).keySet().iterator();
- if (find.hasNext()) {
- TServerInstance current = find.next();
- if (current.getHost().equals(last.getHost())) {
- return current;
- }
- }
- }
-
- // The strategy here is to walk through the locations and hand them back, one at a time
- // Grab an iterator off of the set of options; use a new iterator if it hands back something not
- // in the current list.
- if (assignments == null || !assignments.hasNext()) {
- assignments = randomize(locations.keySet()).iterator();
- }
- TServerInstance result = assignments.next();
- if (!locations.containsKey(result)) {
- assignments = null;
- return randomize(locations.keySet()).iterator().next();
- }
- return result;
- }
-
- static class ServerCounts implements Comparable<ServerCounts> {
- public final TServerInstance server;
- public int count;
- public final TabletServerStatus status;
-
- ServerCounts(int count, TServerInstance server, TabletServerStatus status) {
- this.count = count;
- this.server = server;
- this.status = status;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(server) + count;
- }
-
- @Override
- public boolean equals(Object obj) {
- return obj == this
- || (obj != null && obj instanceof ServerCounts && compareTo((ServerCounts) obj) == 0);
- }
-
- @Override
- public int compareTo(ServerCounts obj) {
- int result = count - obj.count;
- if (result == 0) {
- return server.compareTo(obj.server);
- }
- return result;
- }
- }
-
- public boolean getMigrations(Map<TServerInstance,TabletServerStatus> current,
- List<TabletMigration> result) {
- boolean moreBalancingNeeded = false;
- try {
- // no moves possible
- if (current.size() < 2) {
- return false;
- }
- final Map<TableId,Map<KeyExtent,TabletStats>> donerTabletStats = new HashMap<>();
-
- // Sort by total number of online tablets, per server
- int total = 0;
- ArrayList<ServerCounts> totals = new ArrayList<>();
- for (Entry<TServerInstance,TabletServerStatus> entry : current.entrySet()) {
- int serverTotal = 0;
- if (entry.getValue() != null && entry.getValue().tableMap != null) {
- for (Entry<String,TableInfo> e : entry.getValue().tableMap.entrySet()) {
- /**
- * The check below was on entry.getKey(), but that resolves to a tabletserver not a
- * tablename. Believe it should be e.getKey() which is a tablename
- */
- if (tableToBalance == null || tableToBalance.canonical().equals(e.getKey())) {
- serverTotal += e.getValue().onlineTablets;
- }
- }
- }
- totals.add(new ServerCounts(serverTotal, entry.getKey(), entry.getValue()));
- total += serverTotal;
- }
-
- // order from low to high
- totals.sort(Collections.reverseOrder());
- int even = total / totals.size();
- int numServersOverEven = total % totals.size();
-
- // Move tablets from the servers with too many to the servers with
- // the fewest but only nominate tablets to move once. This allows us
- // to fill new servers with tablets from a mostly balanced server
- // very quickly. However, it may take several balancing passes to move
- // tablets from one hugely overloaded server to many slightly
- // under-loaded servers.
- int end = totals.size() - 1;
- int movedAlready = 0;
- int tooManyIndex = 0;
- while (tooManyIndex < end) {
- ServerCounts tooMany = totals.get(tooManyIndex);
- int goal = even;
- if (tooManyIndex < numServersOverEven) {
- goal++;
- }
- int needToUnload = tooMany.count - goal;
- ServerCounts tooLittle = totals.get(end);
- int needToLoad = goal - tooLittle.count - movedAlready;
- if (needToUnload < 1 && needToLoad < 1) {
- break;
- }
- if (needToUnload >= needToLoad) {
- result.addAll(move(tooMany, tooLittle, needToLoad, donerTabletStats));
- end--;
- movedAlready = 0;
- } else {
- result.addAll(move(tooMany, tooLittle, needToUnload, donerTabletStats));
- movedAlready += needToUnload;
- }
- if (needToUnload > needToLoad) {
- moreBalancingNeeded = true;
- } else {
- tooManyIndex++;
- donerTabletStats.clear();
- }
- }
-
- } finally {
- log.trace("balance ended with {} migrations", result.size());
- }
- return moreBalancingNeeded;
- }
-
- /**
- * Select a tablet based on differences between table loads; if the loads are even, use the
- * busiest table
- */
- List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count,
- Map<TableId,Map<KeyExtent,TabletStats>> donerTabletStats) {
-
- if (count == 0) {
- return Collections.emptyList();
- }
-
- List<TabletMigration> result = new ArrayList<>();
- // Copy counts so we can update them as we propose migrations
- Map<TableId,Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
- Map<TableId,Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
-
- for (int i = 0; i < count; i++) {
- TableId table;
- Integer tooLittleCount;
- if (tableToBalance == null) {
- // find a table to migrate
- // look for an uneven table count
- int biggestDifference = 0;
- TableId biggestDifferenceTable = null;
- for (var tableEntry : tooMuchMap.entrySet()) {
- TableId tableID = tableEntry.getKey();
- tooLittleMap.putIfAbsent(tableID, 0);
- int diff = tableEntry.getValue() - tooLittleMap.get(tableID);
- if (diff > biggestDifference) {
- biggestDifference = diff;
- biggestDifferenceTable = tableID;
- }
- }
- if (biggestDifference < 2) {
- table = busiest(tooMuch.status.tableMap);
- } else {
- table = biggestDifferenceTable;
- }
- } else {
- // just balance the given table
- table = tableToBalance;
- }
- Map<KeyExtent,TabletStats> onlineTabletsForTable = donerTabletStats.get(table);
- try {
- if (onlineTabletsForTable == null) {
- onlineTabletsForTable = new HashMap<>();
- List<TabletStats> stats = getOnlineTabletsForTable(tooMuch.server, table);
- if (stats == null) {
- log.warn("Unable to find tablets to move");
- return result;
- }
- for (TabletStats stat : stats) {
- onlineTabletsForTable.put(KeyExtent.fromThrift(stat.extent), stat);
- }
- donerTabletStats.put(table, onlineTabletsForTable);
- }
- } catch (Exception ex) {
- log.error("Unable to select a tablet to move", ex);
- return result;
- }
- KeyExtent extent = selectTablet(onlineTabletsForTable);
- onlineTabletsForTable.remove(extent);
- if (extent == null) {
- return result;
- }
- tooMuchMap.put(table, tooMuchMap.get(table) - 1);
- /**
- * If a table grows from 1 tablet then tooLittleMap.get(table) can return a null, since there
- * is only one tabletserver that holds all of the tablets. Here we check to see if in fact
- * that is the case and if so set the value to 0.
- */
- tooLittleCount = tooLittleMap.get(table);
- if (tooLittleCount == null) {
- tooLittleCount = 0;
- }
- tooLittleMap.put(table, tooLittleCount + 1);
- tooMuch.count--;
- tooLittle.count++;
- result.add(new TabletMigration(extent, tooMuch.server, tooLittle.server));
- }
- return result;
- }
-
- static Map<TableId,Integer> tabletCountsPerTable(TabletServerStatus status) {
- Map<TableId,Integer> result = new HashMap<>();
- if (status != null && status.tableMap != null) {
- Map<String,TableInfo> tableMap = status.tableMap;
- for (Entry<String,TableInfo> entry : tableMap.entrySet()) {
- result.put(TableId.of(entry.getKey()), entry.getValue().onlineTablets);
- }
- }
- return result;
- }
-
- static KeyExtent selectTablet(Map<KeyExtent,TabletStats> extents) {
- if (extents.isEmpty()) {
- return null;
- }
- KeyExtent mostRecentlySplit = null;
- long splitTime = 0;
- for (Entry<KeyExtent,TabletStats> entry : extents.entrySet()) {
- if (entry.getValue().splitCreationTime >= splitTime) {
- splitTime = entry.getValue().splitCreationTime;
- mostRecentlySplit = entry.getKey();
- }
- }
- return mostRecentlySplit;
- }
-
- // define what it means for a tablet to be busy
- private static TableId busiest(Map<String,TableInfo> tables) {
- TableId result = null;
- double busiest = Double.NEGATIVE_INFINITY;
- for (Entry<String,TableInfo> entry : tables.entrySet()) {
- TableInfo info = entry.getValue();
- double busy = info.ingestRate + info.queryRate;
- if (busy > busiest) {
- busiest = busy;
- result = TableId.of(entry.getKey());
- }
- }
- return result;
- }
-
- @Override
- public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
- Map<KeyExtent,TServerInstance> unassigned, Map<KeyExtent,TServerInstance> assignments) {
- for (Entry<KeyExtent,TServerInstance> entry : unassigned.entrySet()) {
- assignments.put(entry.getKey(), getAssignment(current, entry.getValue()));
- }
- }
-
- private static final NoTservers NO_SERVERS = new NoTservers(log);
-
- protected final OutstandingMigrations outstandingMigrations = new OutstandingMigrations(log);
-
- @Override
- public long balance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
- // do we have any servers?
- if (current.isEmpty()) {
- constraintNotMet(NO_SERVERS);
- } else {
- // Don't migrate if we have migrations in progress
- if (migrations.isEmpty()) {
- resetBalancerErrors();
- if (getMigrations(current, migrationsOut)) {
- return 1_000;
- }
- } else {
- outstandingMigrations.migrations = migrations;
- constraintNotMet(outstandingMigrations);
- }
- }
- return 5_000;
- }
-
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/GroupBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/GroupBalancer.java
deleted file mode 100644
index edf5b4ad9e..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/GroupBalancer.java
+++ /dev/null
@@ -1,751 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
-import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.function.Function;
-
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
-import org.apache.accumulo.core.util.ComparablePair;
-import org.apache.accumulo.core.util.MapCounter;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.HashBasedTable;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Multimap;
-
-/**
- * A balancer that evenly spreads groups of tablets across all tablet server. This balancer
- * accomplishes the following two goals :
- *
- * <ul>
- * <li>Evenly spreads each group across all tservers.
- * <li>Minimizes the total number of groups on each tserver.
- * </ul>
- *
- * <p>
- * To use this balancer you must extend it and implement {@link #getPartitioner()}. See
- * {@link RegexGroupBalancer} as an example.
- *
- * @deprecated since 2.1.0. Use {@link org.apache.accumulo.core.spi.balancer.GroupBalancer} instead.
- */
-@Deprecated(since = "2.1.0")
-public abstract class GroupBalancer extends TabletBalancer {
-
- private final TableId tableId;
- private long lastRun = 0;
-
- /**
- * @return A function that groups tablets into named groups.
- */
- protected abstract Function<KeyExtent,String> getPartitioner();
-
- public GroupBalancer(TableId tableId) {
- this.tableId = tableId;
-
- LoggerFactory.getLogger(getClass().getName())
- .warn("{} has been deprecated and will be "
- + "removed in a future release. Please update your configuration to use the equivalent "
- + "{} instead.", getClass().getName(),
- org.apache.accumulo.core.spi.balancer.GroupBalancer.class.getName());
-
- }
-
- protected Map<KeyExtent,TServerInstance> getLocationProvider() {
- Map<KeyExtent,TServerInstance> tablets = new LinkedHashMap<>();
- for (var tm : TabletsMetadata.builder(context).forTable(tableId).fetch(LOCATION, PREV_ROW)
- .build()) {
- tablets.put(tm.getExtent(), tm.getLocation());
- }
- return tablets;
- }
-
- /**
- * The amount of time to wait between balancing.
- */
- protected long getWaitTime() {
- return 60000;
- }
-
- /**
- * The maximum number of migrations to perform in a single pass.
- */
- protected int getMaxMigrations() {
- return 1000;
- }
-
- /**
- * @return Examine current tserver and migrations and return true if balancing should occur.
- */
- protected boolean shouldBalance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations) {
-
- if (current.size() < 2) {
- return false;
- }
-
- for (KeyExtent keyExtent : migrations) {
- if (keyExtent.tableId().equals(tableId)) {
- return false;
- }
- }
-
- return true;
- }
-
- @Override
- public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
- Map<KeyExtent,TServerInstance> unassigned, Map<KeyExtent,TServerInstance> assignments) {
-
- if (current.isEmpty()) {
- return;
- }
-
- Function<KeyExtent,String> partitioner = getPartitioner();
-
- List<ComparablePair<String,KeyExtent>> tabletsByGroup = new ArrayList<>();
- for (Entry<KeyExtent,TServerInstance> entry : unassigned.entrySet()) {
- TServerInstance last = entry.getValue();
- if (last != null) {
- // Maintain locality
- String fakeSessionID = " ";
- TServerInstance simple = new TServerInstance(last.getHostAndPort(), fakeSessionID);
- Iterator<TServerInstance> find = current.tailMap(simple).keySet().iterator();
- if (find.hasNext()) {
- TServerInstance tserver = find.next();
- if (tserver.getHost().equals(last.getHost())) {
- assignments.put(entry.getKey(), tserver);
- continue;
- }
- }
- }
-
- tabletsByGroup.add(new ComparablePair<>(partitioner.apply(entry.getKey()), entry.getKey()));
- }
-
- Collections.sort(tabletsByGroup);
-
- Iterator<TServerInstance> tserverIter = Iterators.cycle(current.keySet());
- for (ComparablePair<String,KeyExtent> pair : tabletsByGroup) {
- KeyExtent ke = pair.getSecond();
- assignments.put(ke, tserverIter.next());
- }
-
- }
-
- @Override
- public long balance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
-
- // The terminology extra and expected are used in this code. Expected tablets is the number of
- // tablets a tserver must have for a given group and is
- // numInGroup/numTservers. Extra tablets are any tablets more than the number expected for a
- // given group. If numInGroup % numTservers > 0, then a tserver
- // may have one extra tablet for a group.
- //
- // Assume we have 4 tservers and group A has 11 tablets.
- // * expected tablets : group A is expected to have 2 tablets on each tservers
- // * extra tablets : group A may have an additional tablet on each tserver. Group A has a total
- // of 3 extra tablets.
- //
- // This balancer also evens out the extra tablets across all groups. The terminology
- // extraExpected and extraExtra is used to describe these tablets.
- // ExtraExpected is totalExtra/numTservers. ExtraExtra is totalExtra%numTservers. Each tserver
- // should have at least expectedExtra extra tablets and at most
- // one extraExtra tablets. All extra tablets on a tserver must be from different groups.
- //
- // Assume we have 6 tservers and three groups (G1, G2, G3) with 9 tablets each. Each tserver is
- // expected to have one tablet from each group and could
- // possibly have 2 tablets from a group. Below is an illustration of an ideal balancing of extra
- // tablets. To understand the illustration, the first column
- // shows tserver T1 with 2 tablets from G1, 1 tablet from G2, and two tablets from G3. EE means
- // empty, put it there so eclipse formatting would not mess up
- // table.
- //
- // T1 | T2 | T3 | T4 | T5 | T6
- // ---+----+----+----+----+-----
- // G3 | G2 | G3 | EE | EE | EE <-- extra extra tablets
- // G1 | G1 | G1 | G2 | G3 | G2 <-- extra expected tablets.
- // G1 | G1 | G1 | G1 | G1 | G1 <-- expected tablets for group 1
- // G2 | G2 | G2 | G2 | G2 | G2 <-- expected tablets for group 2
- // G3 | G3 | G3 | G3 | G3 | G3 <-- expected tablets for group 3
- //
- // Do not want to balance the extra tablets like the following. There are two problem with this.
- // First extra tablets are not evenly spread. Since there are
- // a total of 9 extra tablets, every tserver is expected to have at least one extra tablet.
- // Second tserver T1 has two extra tablet for group G1. This
- // violates the principal that a tserver can only have one extra tablet for a given group.
- //
- // T1 | T2 | T3 | T4 | T5 | T6
- // ---+----+----+----+----+-----
- // G1 | EE | EE | EE | EE | EE <--- one extra tablets from group 1
- // G3 | G3 | G3 | EE | EE | EE <--- three extra tablets from group 3
- // G2 | G2 | G2 | EE | EE | EE <--- three extra tablets from group 2
- // G1 | G1 | EE | EE | EE | EE <--- two extra tablets from group 1
- // G1 | G1 | G1 | G1 | G1 | G1 <-- expected tablets for group 1
- // G2 | G2 | G2 | G2 | G2 | G2 <-- expected tablets for group 2
- // G3 | G3 | G3 | G3 | G3 | G3 <-- expected tablets for group 3
-
- if (!shouldBalance(current, migrations)) {
- return 5000;
- }
-
- if (System.currentTimeMillis() - lastRun < getWaitTime()) {
- return 5000;
- }
-
- MapCounter<String> groupCounts = new MapCounter<>();
- Map<TServerInstance,TserverGroupInfo> tservers = new HashMap<>();
-
- for (TServerInstance tsi : current.keySet()) {
- tservers.put(tsi, new TserverGroupInfo(tsi));
- }
-
- Function<KeyExtent,String> partitioner = getPartitioner();
-
- // collect stats about current state
- for (var tablet : getLocationProvider().entrySet()) {
- String group = partitioner.apply(tablet.getKey());
- var loc = tablet.getValue();
-
- if (loc == null || !tservers.containsKey(loc)) {
- return 5000;
- }
-
- groupCounts.increment(group, 1);
- TserverGroupInfo tgi = tservers.get(loc);
- tgi.addGroup(group);
- }
-
- Map<String,Integer> expectedCounts = new HashMap<>();
-
- int totalExtra = 0;
- for (String group : groupCounts.keySet()) {
- int groupCount = groupCounts.getInt(group);
- totalExtra += groupCount % current.size();
- expectedCounts.put(group, (groupCount / current.size()));
- }
-
- // The number of extra tablets from all groups that each tserver must have.
- int expectedExtra = totalExtra / current.size();
- int maxExtraGroups = expectedExtra + 1;
-
- expectedCounts = Collections.unmodifiableMap(expectedCounts);
- tservers = Collections.unmodifiableMap(tservers);
-
- for (TserverGroupInfo tgi : tservers.values()) {
- tgi.finishedAdding(expectedCounts);
- }
-
- Moves moves = new Moves();
-
- // The order of the following steps is important, because as ordered each step should not move
- // any tablets moved by a previous step.
- balanceExpected(tservers, moves);
- if (moves.size() < getMaxMigrations()) {
- balanceExtraExpected(tservers, expectedExtra, moves);
- if (moves.size() < getMaxMigrations()) {
- boolean cont = balanceExtraMultiple(tservers, maxExtraGroups, moves);
- if (cont && moves.size() < getMaxMigrations()) {
- balanceExtraExtra(tservers, maxExtraGroups, moves);
- }
- }
- }
-
- populateMigrations(tservers.keySet(), migrationsOut, moves);
-
- lastRun = System.currentTimeMillis();
-
- return 5000;
- }
-
- static class TserverGroupInfo {
-
- private Map<String,Integer> expectedCounts;
- private final Map<String,MutableInt> initialCounts = new HashMap<>();
- private final Map<String,Integer> extraCounts = new HashMap<>();
- private final Map<String,Integer> expectedDeficits = new HashMap<>();
-
- private final TServerInstance tsi;
- private boolean finishedAdding = false;
-
- TserverGroupInfo(TServerInstance tsi) {
- this.tsi = tsi;
- }
-
- public void addGroup(String group) {
- checkState(!finishedAdding);
-
- MutableInt mi = initialCounts.get(group);
- if (mi == null) {
- mi = new MutableInt();
- initialCounts.put(group, mi);
- }
-
- mi.increment();
- }
-
- public void finishedAdding(Map<String,Integer> expectedCounts) {
- checkState(!finishedAdding);
- finishedAdding = true;
- this.expectedCounts = expectedCounts;
-
- for (Entry<String,Integer> entry : expectedCounts.entrySet()) {
- String group = entry.getKey();
- int expected = entry.getValue();
-
- MutableInt count = initialCounts.get(group);
- int num = count == null ? 0 : count.intValue();
-
- if (num < expected) {
- expectedDeficits.put(group, expected - num);
- } else if (num > expected) {
- extraCounts.put(group, num - expected);
- }
- }
-
- }
-
- public void moveOff(String group, int num) {
- checkArgument(num > 0);
- checkState(finishedAdding);
-
- Integer extraCount = extraCounts.get(group);
-
- // don't wrap precondition check due to https://github.com/spotbugs/spotbugs/issues/462
- String formatString = "group=%s num=%s extraCount=%s";
- checkArgument(extraCount != null && extraCount >= num, formatString, group, num, extraCount);
-
- MutableInt initialCount = initialCounts.get(group);
-
- checkArgument(initialCount.intValue() >= num);
-
- initialCount.subtract(num);
-
- if (extraCount - num == 0) {
- extraCounts.remove(group);
- } else {
- extraCounts.put(group, extraCount - num);
- }
- }
-
- public void moveTo(String group, int num) {
- checkArgument(num > 0);
- checkArgument(expectedCounts.containsKey(group));
- checkState(finishedAdding);
-
- Integer deficit = expectedDeficits.get(group);
- if (deficit != null) {
- if (num >= deficit) {
- expectedDeficits.remove(group);
- num -= deficit;
- } else {
- expectedDeficits.put(group, deficit - num);
- num = 0;
- }
- }
-
- if (num > 0) {
- Integer extra = extraCounts.get(group);
- if (extra == null) {
- extra = 0;
- }
-
- extraCounts.put(group, extra + num);
- }
-
- // TODO could check extra constraints
- }
-
- public Map<String,Integer> getExpectedDeficits() {
- checkState(finishedAdding);
- return Collections.unmodifiableMap(expectedDeficits);
- }
-
- public Map<String,Integer> getExtras() {
- checkState(finishedAdding);
- return Collections.unmodifiableMap(extraCounts);
- }
-
- public TServerInstance getTserverInstance() {
- return tsi;
- }
-
- @Override
- public int hashCode() {
- return tsi.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof TserverGroupInfo) {
- TserverGroupInfo otgi = (TserverGroupInfo) o;
- return tsi.equals(otgi.tsi);
- }
-
- return false;
- }
-
- @Override
- public String toString() {
- return tsi.getHostPortSession();
- }
-
- }
-
- private static class Move {
- TserverGroupInfo dest;
- int count;
-
- public Move(TserverGroupInfo dest, int num) {
- this.dest = dest;
- this.count = num;
- }
- }
-
- private static class Moves {
-
- private final HashBasedTable<TServerInstance,String,List<Move>> moves = HashBasedTable.create();
- private int totalMoves = 0;
-
- public void move(String group, int num, TserverGroupInfo src, TserverGroupInfo dest) {
- checkArgument(num > 0);
- checkArgument(!src.equals(dest));
-
- src.moveOff(group, num);
- dest.moveTo(group, num);
-
- List<Move> srcMoves = moves.get(src.getTserverInstance(), group);
- if (srcMoves == null) {
- srcMoves = new ArrayList<>();
- moves.put(src.getTserverInstance(), group, srcMoves);
- }
-
- srcMoves.add(new Move(dest, num));
- totalMoves += num;
- }
-
- public TServerInstance removeMove(TServerInstance src, String group) {
- List<Move> srcMoves = moves.get(src, group);
- if (srcMoves == null) {
- return null;
- }
-
- Move move = srcMoves.get(srcMoves.size() - 1);
- TServerInstance ret = move.dest.getTserverInstance();
- totalMoves--;
-
- move.count--;
- if (move.count == 0) {
- srcMoves.remove(srcMoves.size() - 1);
- if (srcMoves.isEmpty()) {
- moves.remove(src, group);
- }
- }
-
- return ret;
- }
-
- public int size() {
- return totalMoves;
- }
- }
-
- private void balanceExtraExtra(Map<TServerInstance,TserverGroupInfo> tservers, int maxExtraGroups,
- Moves moves) {
- HashBasedTable<String,TServerInstance,TserverGroupInfo> surplusExtra = HashBasedTable.create();
- for (TserverGroupInfo tgi : tservers.values()) {
- Map<String,Integer> extras = tgi.getExtras();
- if (extras.size() > maxExtraGroups) {
- for (String group : extras.keySet()) {
- surplusExtra.put(group, tgi.getTserverInstance(), tgi);
- }
- }
- }
-
- ArrayList<Pair<String,TServerInstance>> serversGroupsToRemove = new ArrayList<>();
- ArrayList<TServerInstance> serversToRemove = new ArrayList<>();
-
- for (TserverGroupInfo destTgi : tservers.values()) {
- if (surplusExtra.isEmpty()) {
- break;
- }
-
- Map<String,Integer> extras = destTgi.getExtras();
- if (extras.size() < maxExtraGroups) {
- serversToRemove.clear();
- serversGroupsToRemove.clear();
- for (String group : surplusExtra.rowKeySet()) {
- if (!extras.containsKey(group)) {
- TserverGroupInfo srcTgi = surplusExtra.row(group).values().iterator().next();
-
- moves.move(group, 1, srcTgi, destTgi);
-
- if (srcTgi.getExtras().size() <= maxExtraGroups) {
- serversToRemove.add(srcTgi.getTserverInstance());
- } else {
- serversGroupsToRemove.add(new Pair<>(group, srcTgi.getTserverInstance()));
- }
-
- if (destTgi.getExtras().size() >= maxExtraGroups
- || moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
-
- if (!serversToRemove.isEmpty()) {
- surplusExtra.columnKeySet().removeAll(serversToRemove);
- }
-
- for (Pair<String,TServerInstance> pair : serversGroupsToRemove) {
- surplusExtra.remove(pair.getFirst(), pair.getSecond());
- }
-
- if (moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
- }
-
- private boolean balanceExtraMultiple(Map<TServerInstance,TserverGroupInfo> tservers,
- int maxExtraGroups, Moves moves) {
- Multimap<String,TserverGroupInfo> extraMultiple = HashMultimap.create();
-
- for (TserverGroupInfo tgi : tservers.values()) {
- Map<String,Integer> extras = tgi.getExtras();
- for (Entry<String,Integer> entry : extras.entrySet()) {
- if (entry.getValue() > 1) {
- extraMultiple.put(entry.getKey(), tgi);
- }
- }
- }
-
- balanceExtraMultiple(tservers, maxExtraGroups, moves, extraMultiple, false);
- if (moves.size() < getMaxMigrations() && !extraMultiple.isEmpty()) {
- // no place to move so must exceed maxExtra temporarily... subsequent balancer calls will
- // smooth things out
- balanceExtraMultiple(tservers, maxExtraGroups, moves, extraMultiple, true);
- return false;
- } else {
- return true;
- }
- }
-
- private void balanceExtraMultiple(Map<TServerInstance,TserverGroupInfo> tservers,
- int maxExtraGroups, Moves moves, Multimap<String,TserverGroupInfo> extraMultiple,
- boolean alwaysAdd) {
-
- ArrayList<Pair<String,TserverGroupInfo>> serversToRemove = new ArrayList<>();
- for (TserverGroupInfo destTgi : tservers.values()) {
- Map<String,Integer> extras = destTgi.getExtras();
- if (alwaysAdd || extras.size() < maxExtraGroups) {
- serversToRemove.clear();
- for (String group : extraMultiple.keySet()) {
- if (!extras.containsKey(group)) {
- Collection<TserverGroupInfo> sources = extraMultiple.get(group);
- Iterator<TserverGroupInfo> iter = sources.iterator();
- TserverGroupInfo srcTgi = iter.next();
-
- int num = srcTgi.getExtras().get(group);
-
- moves.move(group, 1, srcTgi, destTgi);
-
- if (num == 2) {
- serversToRemove.add(new Pair<>(group, srcTgi));
- }
-
- if (destTgi.getExtras().size() >= maxExtraGroups
- || moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
-
- for (Pair<String,TserverGroupInfo> pair : serversToRemove) {
- extraMultiple.remove(pair.getFirst(), pair.getSecond());
- }
-
- if (extraMultiple.isEmpty() || moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
- }
-
- private void balanceExtraExpected(Map<TServerInstance,TserverGroupInfo> tservers,
- int expectedExtra, Moves moves) {
-
- HashBasedTable<String,TServerInstance,TserverGroupInfo> extraSurplus = HashBasedTable.create();
-
- for (TserverGroupInfo tgi : tservers.values()) {
- Map<String,Integer> extras = tgi.getExtras();
- if (extras.size() > expectedExtra) {
- for (String group : extras.keySet()) {
- extraSurplus.put(group, tgi.getTserverInstance(), tgi);
- }
- }
- }
-
- ArrayList<TServerInstance> emptyServers = new ArrayList<>();
- ArrayList<Pair<String,TServerInstance>> emptyServerGroups = new ArrayList<>();
- for (TserverGroupInfo destTgi : tservers.values()) {
- if (extraSurplus.isEmpty()) {
- break;
- }
-
- Map<String,Integer> extras = destTgi.getExtras();
- if (extras.size() < expectedExtra) {
- emptyServers.clear();
- emptyServerGroups.clear();
- nextGroup: for (String group : extraSurplus.rowKeySet()) {
- if (!extras.containsKey(group)) {
- Iterator<TserverGroupInfo> iter = extraSurplus.row(group).values().iterator();
- TserverGroupInfo srcTgi = iter.next();
-
- while (srcTgi.getExtras().size() <= expectedExtra) {
- if (iter.hasNext()) {
- srcTgi = iter.next();
- } else {
- continue nextGroup;
- }
- }
-
- moves.move(group, 1, srcTgi, destTgi);
-
- if (srcTgi.getExtras().size() <= expectedExtra) {
- emptyServers.add(srcTgi.getTserverInstance());
- } else if (srcTgi.getExtras().get(group) == null) {
- emptyServerGroups.add(new Pair<>(group, srcTgi.getTserverInstance()));
- }
-
- if (destTgi.getExtras().size() >= expectedExtra || moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
-
- if (!emptyServers.isEmpty()) {
- extraSurplus.columnKeySet().removeAll(emptyServers);
- }
-
- for (Pair<String,TServerInstance> pair : emptyServerGroups) {
- extraSurplus.remove(pair.getFirst(), pair.getSecond());
- }
-
- if (moves.size() >= getMaxMigrations()) {
- break;
- }
- }
- }
- }
-
- private void balanceExpected(Map<TServerInstance,TserverGroupInfo> tservers, Moves moves) {
- Multimap<String,TserverGroupInfo> groupDefecits = HashMultimap.create();
- Multimap<String,TserverGroupInfo> groupSurplus = HashMultimap.create();
-
- for (TserverGroupInfo tgi : tservers.values()) {
- for (String group : tgi.getExpectedDeficits().keySet()) {
- groupDefecits.put(group, tgi);
- }
-
- for (String group : tgi.getExtras().keySet()) {
- groupSurplus.put(group, tgi);
- }
- }
-
- for (String group : groupDefecits.keySet()) {
- Collection<TserverGroupInfo> defecitServers = groupDefecits.get(group);
- for (TserverGroupInfo defecitTsi : defecitServers) {
- int numToMove = defecitTsi.getExpectedDeficits().get(group);
-
- Iterator<TserverGroupInfo> surplusIter = groupSurplus.get(group).iterator();
- while (numToMove > 0) {
- TserverGroupInfo surplusTsi = surplusIter.next();
-
- int available = surplusTsi.getExtras().get(group);
-
- if (numToMove >= available) {
- surplusIter.remove();
- }
-
- int transfer = Math.min(numToMove, available);
-
- numToMove -= transfer;
-
- moves.move(group, transfer, surplusTsi, defecitTsi);
- if (moves.size() >= getMaxMigrations()) {
- return;
- }
- }
- }
- }
- }
-
- private void populateMigrations(Set<TServerInstance> current, List<TabletMigration> migrationsOut,
- Moves moves) {
- if (moves.size() == 0) {
- return;
- }
-
- Function<KeyExtent,String> partitioner = getPartitioner();
-
- for (var tablet : getLocationProvider().entrySet()) {
- String group = partitioner.apply(tablet.getKey());
- var loc = tablet.getValue();
-
- if (loc == null || !current.contains(loc)) {
- migrationsOut.clear();
- return;
- }
-
- TServerInstance dest = moves.removeMove(loc, group);
- if (dest != null) {
- migrationsOut.add(new TabletMigration(tablet.getKey(), loc, dest));
- if (moves.size() == 0) {
- break;
- }
- }
- }
- }
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
deleted file mode 100644
index a965b5fd0a..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ /dev/null
@@ -1,576 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static java.util.concurrent.TimeUnit.HOURS;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.AccumuloConfiguration.Deriver;
-import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * This balancer creates groups of tablet servers using user-provided regular expressions over the
- * tablet server hostnames. Then it delegates to the table balancer to balance the tablets within
- * the resulting group of tablet servers. All tablet servers that do not match a regex are grouped
- * into a default group.<br>
- * Regex properties for this balancer are specified as:<br>
- * <b>table.custom.balancer.host.regex.<tablename>=<regex></b><br>
- * Periodically (default 5m) this balancer will check to see if a tablet server is hosting tablets
- * that it should not be according to the regex configuration. If this occurs then the offending
- * tablets will be reassigned. This would cover the case where the configuration is changed and the
- * manager is restarted while the tablet servers are up. To change the out of bounds check time
- * period, set the following property:<br>
- * <b>table.custom.balancer.host.regex.oob.period=5m</b><br>
- * Regex matching can be based on either the host name (default) or host ip address. To set this
- * balancer to match the regular expressions to the tablet server IP address, then set the following
- * property:<br>
- * <b>table.custom.balancer.host.regex.is.ip=true</b><br>
- * It's possible that this balancer may create a lot of migrations. To limit the number of
- * migrations that are created during a balance call, set the following property (default 250):<br>
- * <b>table.custom.balancer.host.regex.concurrent.migrations</b> This balancer can continue
- * balancing even if there are outstanding migrations. To limit the number of outstanding migrations
- * in which this balancer will continue balancing, set the following property (default 0):<br>
- * <b>table.custom.balancer.host.regex.max.outstanding.migrations</b>
- *
- * @deprecated since 2.1.0. Use
- * {@link org.apache.accumulo.core.spi.balancer.HostRegexTableLoadBalancer} instead
- */
-@Deprecated(since = "2.1.0")
-public class HostRegexTableLoadBalancer extends TableLoadBalancer {
-
- private static final SecureRandom random = new SecureRandom();
- private static final String PROP_PREFIX = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey();
-
- private static final Logger LOG = LoggerFactory.getLogger(HostRegexTableLoadBalancer.class);
- public static final String HOST_BALANCER_PREFIX = PROP_PREFIX + "balancer.host.regex.";
- public static final String HOST_BALANCER_OOB_CHECK_KEY =
- PROP_PREFIX + "balancer.host.regex.oob.period";
- private static final String HOST_BALANCER_OOB_DEFAULT = "5m";
- public static final String HOST_BALANCER_REGEX_USING_IPS_KEY =
- PROP_PREFIX + "balancer.host.regex.is.ip";
- public static final String HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY =
- PROP_PREFIX + "balancer.host.regex.concurrent.migrations";
- private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
- protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
- private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
- public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY =
- PROP_PREFIX + "balancer.host.regex.max.outstanding.migrations";
-
- private static Map<String,String> getRegexes(AccumuloConfiguration aconf) {
- Map<String,String> regexes = new HashMap<>();
- Map<String,String> customProps =
- aconf.getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
-
- if (customProps != null && !customProps.isEmpty()) {
- for (Entry<String,String> customProp : customProps.entrySet()) {
- if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
- if (customProp.getKey().equals(HOST_BALANCER_OOB_CHECK_KEY)
- || customProp.getKey().equals(HOST_BALANCER_REGEX_USING_IPS_KEY)
- || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY)
- || customProp.getKey().equals(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY)) {
- continue;
- }
- String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
- String regex = customProp.getValue();
- regexes.put(tableName, regex);
- }
- }
- }
-
- return Map.copyOf(regexes);
- }
-
- /**
- * Host Regex Table Load Balance Config
- */
- static class HrtlbConf {
-
- protected long oobCheckMillis =
- ConfigurationTypeHelper.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
- private int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
- private int maxOutstandingMigrations = DEFAULT_OUTSTANDING_MIGRATIONS;
- private boolean isIpBasedRegex = false;
- private Map<String,String> regexes;
- private Map<String,Pattern> poolNameToRegexPattern = null;
-
- HrtlbConf(AccumuloConfiguration aconf) {
- System.out.println("building hrtlb conf");
- String oobProperty = aconf.get(HOST_BALANCER_OOB_CHECK_KEY);
- if (oobProperty != null) {
- oobCheckMillis = ConfigurationTypeHelper.getTimeInMillis(oobProperty);
- }
- String ipBased = aconf.get(HOST_BALANCER_REGEX_USING_IPS_KEY);
- if (ipBased != null) {
- isIpBasedRegex = Boolean.parseBoolean(ipBased);
- }
- String migrations = aconf.get(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY);
- if (migrations != null) {
- maxTServerMigrations = Integer.parseInt(migrations);
- }
- String outstanding = aconf.get(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY);
- if (outstanding != null) {
- maxOutstandingMigrations = Integer.parseInt(outstanding);
- }
-
- this.regexes = getRegexes(aconf);
-
- Map<String,Pattern> poolNameToRegexPatternBuilder = new HashMap<>();
- regexes.forEach((k, v) -> {
- poolNameToRegexPatternBuilder.put(k, Pattern.compile(v));
- });
-
- poolNameToRegexPattern = Map.copyOf(poolNameToRegexPatternBuilder);
- }
- }
-
- private static final Set<KeyExtent> EMPTY_MIGRATIONS = Collections.emptySet();
- private volatile long lastOOBCheck = System.currentTimeMillis();
- private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new HashMap<>();
- private final Map<KeyExtent,TabletMigration> migrationsFromLastPass = new HashMap<>();
- private final Map<String,Long> tableToTimeSinceNoMigrations = new HashMap<>();
-
- private Deriver<HrtlbConf> hrtlbConf;
- private LoadingCache<TableId,Deriver<Map<String,String>>> tablesRegExCache;
-
- /**
- * Group the set of current tservers by pool name. Tservers that don't match a regex are put into
- * a default pool. This could be expensive in the terms of the amount of time to recompute the
- * groups, so HOST_BALANCER_POOL_RECHECK_KEY should be specified in the terms of minutes, not
- * seconds or less.
- *
- * @param current map of current tservers
- * @return current servers grouped by pool name, if not a match it is put into a default pool.
- */
- protected synchronized Map<String,SortedMap<TServerInstance,TabletServerStatus>>
- splitCurrentByRegex(SortedMap<TServerInstance,TabletServerStatus> current) {
- LOG.debug("Performing pool recheck - regrouping tablet servers based on regular expressions");
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools = new HashMap<>();
- for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- List<String> poolNames = getPoolNamesForHost(e.getKey().getHost());
- for (String pool : poolNames) {
- SortedMap<TServerInstance,TabletServerStatus> np = newPools.get(pool);
- if (np == null) {
- np = new TreeMap<>(current.comparator());
- newPools.put(pool, np);
- }
- np.put(e.getKey(), e.getValue());
- }
- }
-
- if (newPools.get(DEFAULT_POOL) == null) {
- LOG.warn("Default pool is empty; assigning all tablet servers to the default pool");
- SortedMap<TServerInstance,TabletServerStatus> dp = new TreeMap<>(current.comparator());
- dp.putAll(current);
- newPools.put(DEFAULT_POOL, dp);
- }
-
- pools = newPools;
-
- LOG.trace("Pool to TabletServer mapping:");
- if (LOG.isTraceEnabled()) {
- for (Entry<String,SortedMap<TServerInstance,TabletServerStatus>> e : pools.entrySet()) {
- LOG.trace("\tpool: {} -> tservers: {}", e.getKey(), e.getValue().keySet());
- }
- }
- return pools;
- }
-
- /**
- * Matches host against the regexes and returns the matching pool names
- *
- * @param host tablet server host
- * @return pool names, will return default pool if host matches more no regex
- */
- protected List<String> getPoolNamesForHost(String host) {
- String test = host;
- if (!hrtlbConf.derive().isIpBasedRegex) {
- try {
- test = getNameFromIp(host);
- } catch (UnknownHostException e1) {
- LOG.error("Unable to determine host name for IP: " + host + ", setting to default pool",
- e1);
- return Collections.singletonList(DEFAULT_POOL);
- }
- }
- List<String> pools = new ArrayList<>();
- for (Entry<String,Pattern> e : hrtlbConf.derive().poolNameToRegexPattern.entrySet()) {
- if (e.getValue().matcher(test).matches()) {
- pools.add(e.getKey());
- }
- }
- if (pools.isEmpty()) {
- pools.add(DEFAULT_POOL);
- }
- return pools;
- }
-
- protected String getNameFromIp(String hostIp) throws UnknownHostException {
- return InetAddress.getByName(hostIp).getHostName();
- }
-
- private void checkTableConfig(TableId tableId) {
- Map<String,String> tableRegexes = tablesRegExCache.getUnchecked(tableId).derive();
-
- if (!hrtlbConf.derive().regexes.equals(tableRegexes)) {
- LoggerFactory.getLogger(HostRegexTableLoadBalancer.class).warn(
- "Table id {} has different config than system. The per table config is ignored.",
- tableId);
- }
- }
-
- private Map<TableId,String> createdTableNameMap(Map<String,String> tableIdMap) {
- HashMap<TableId,String> tableNameMap = new HashMap<>();
- tableIdMap.forEach((tableName, tableId) -> {
- tableNameMap.put(TableId.of(tableId), tableName);
- });
- return tableNameMap;
- }
-
- /**
- * Matches table name against pool names, returns matching pool name or DEFAULT_POOL.
- *
- * @param tableName name of table
- * @return tablet server pool name (table name or DEFAULT_POOL)
- */
- protected String getPoolNameForTable(String tableName) {
- if (tableName == null) {
- return DEFAULT_POOL;
- }
- return hrtlbConf.derive().poolNameToRegexPattern.containsKey(tableName) ? tableName
- : DEFAULT_POOL;
- }
-
- @Override
- public String toString() {
- HrtlbConf myConf = hrtlbConf.derive();
- ToStringBuilder buf = new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE);
- buf.append("\nTablet Out Of Bounds Check Interval", myConf.oobCheckMillis);
- buf.append("\nMax Tablet Server Migrations", myConf.maxTServerMigrations);
- buf.append("\nRegular Expressions use IPs", myConf.isIpBasedRegex);
- buf.append("\nPools", myConf.poolNameToRegexPattern);
- return buf.toString();
- }
-
- public Map<String,Pattern> getPoolNameToRegexPattern() {
- return hrtlbConf.derive().poolNameToRegexPattern;
- }
-
- public int getMaxMigrations() {
- return hrtlbConf.derive().maxTServerMigrations;
- }
-
- public int getMaxOutstandingMigrations() {
- return hrtlbConf.derive().maxOutstandingMigrations;
- }
-
- public long getOobCheckMillis() {
- return hrtlbConf.derive().oobCheckMillis;
- }
-
- public boolean isIpBasedRegex() {
- return hrtlbConf.derive().isIpBasedRegex;
- }
-
- @Override
- public void init(ServerContext context) {
- super.init(context);
-
- this.hrtlbConf = context.getConfiguration().newDeriver(HrtlbConf::new);
-
- tablesRegExCache =
- CacheBuilder.newBuilder().expireAfterAccess(1, HOURS).build(new CacheLoader<>() {
- @Override
- public Deriver<Map<String,String>> load(TableId key) throws Exception {
- return context.getTableConfiguration(key)
- .newDeriver(HostRegexTableLoadBalancer::getRegexes);
- }
- });
-
- LOG.info("{}", this);
- }
-
- @Override
- public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
- Map<KeyExtent,TServerInstance> unassigned, Map<KeyExtent,TServerInstance> assignments) {
-
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = splitCurrentByRegex(current);
- // group the unassigned into tables
- Map<TableId,Map<KeyExtent,TServerInstance>> groupedUnassigned = new HashMap<>();
- unassigned.forEach((ke, lastTserver) -> groupedUnassigned
- .computeIfAbsent(ke.tableId(), k -> new HashMap<>()).put(ke, lastTserver));
-
- Map<TableId,String> tableIdToTableName = createdTableNameMap(getTableOperations().tableIdMap());
-
- // Send a view of the current servers to the tables tablet balancer
- for (Entry<TableId,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet()) {
- Map<KeyExtent,TServerInstance> newAssignments = new HashMap<>();
- String tableName = tableIdToTableName.get(e.getKey());
- String poolName = getPoolNameForTable(tableName);
- SortedMap<TServerInstance,TabletServerStatus> currentView = pools.get(poolName);
- if (currentView == null || currentView.isEmpty()) {
- LOG.warn("No tablet servers online for table {}, assigning within default pool", tableName);
- currentView = pools.get(DEFAULT_POOL);
- if (currentView == null) {
- LOG.error(
- "No tablet servers exist in the default pool, unable to assign tablets for table {}",
- tableName);
- continue;
- }
- }
- LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers {}",
- e.getValue().size(), tableName, currentView.keySet());
- getBalancerForTable(e.getKey()).getAssignments(currentView, e.getValue(), newAssignments);
- assignments.putAll(newAssignments);
- }
- }
-
- @Override
- public long balance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
- long minBalanceTime = 20_000;
- // Iterate over the tables and balance each of them
- TableOperations t = getTableOperations();
- if (t == null) {
- return minBalanceTime;
- }
-
- Map<String,String> tableIdMap = t.tableIdMap();
- Map<TableId,String> tableIdToTableName = createdTableNameMap(tableIdMap);
- tableIdToTableName.keySet().forEach(this::checkTableConfig);
-
- long now = System.currentTimeMillis();
-
- HrtlbConf myConf = hrtlbConf.derive();
-
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped =
- splitCurrentByRegex(current);
- if ((now - this.lastOOBCheck) > myConf.oobCheckMillis) {
- try {
- // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
- for (String table : tableIdMap.keySet()) {
- LOG.debug("Checking for out of bounds tablets for table {}", table);
- String tablePoolName = getPoolNameForTable(table);
- for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- // pool names are the same as table names, except in the DEFAULT case.
- // If this table is assigned to a pool for this host, then move on.
- List<String> hostPools = getPoolNamesForHost(e.getKey().getHost());
- if (hostPools.contains(tablePoolName)) {
- continue;
- }
- String tid = tableIdMap.get(table);
- if (tid == null) {
- LOG.warn("Unable to check for out of bounds tablets for table {},"
- + " it may have been deleted or renamed.", table);
- continue;
- }
- try {
- List<TabletStats> outOfBoundsTablets =
- getOnlineTabletsForTable(e.getKey(), TableId.of(tid));
- if (outOfBoundsTablets == null) {
- continue;
- }
- for (TabletStats ts : outOfBoundsTablets) {
- KeyExtent ke = KeyExtent.fromThrift(ts.getExtent());
- if (migrations.contains(ke)) {
- LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
- continue;
- }
- String poolName = getPoolNameForTable(table);
- SortedMap<TServerInstance,TabletServerStatus> currentView =
- currentGrouped.get(poolName);
- if (currentView != null) {
- int skip = random.nextInt(currentView.size());
- Iterator<TServerInstance> iter = currentView.keySet().iterator();
- for (int i = 0; i < skip; i++) {
- iter.next();
- }
- TServerInstance nextTS = iter.next();
- LOG.info("Tablet {} is currently outside the bounds of the"
- + " regex, migrating from {} to {}", ke, e.getKey(), nextTS);
- migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
- if (migrationsOut.size() >= myConf.maxTServerMigrations) {
- break;
- }
- } else {
- LOG.warn("No tablet servers online for pool {}, unable to"
- + " migrate out of bounds tablets", poolName);
- }
- }
- } catch (TException e1) {
- LOG.error("Error in OOB check getting tablets for table {} from server {} {}", tid,
- e.getKey().getHost(), e);
- }
- }
- }
- } finally {
- // this could have taken a while...get a new time
- this.lastOOBCheck = System.currentTimeMillis();
- }
- }
-
- if (!migrationsOut.isEmpty()) {
- LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
- LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
- return minBalanceTime;
- }
-
- if (migrations != null && !migrations.isEmpty()) {
- if (migrations.size() >= myConf.maxOutstandingMigrations) {
- LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Sample up to 10 outstanding migrations: {}", limitTen(migrations));
- }
- return minBalanceTime;
- }
-
- LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Sample up to 10 outstanding migrations: {}", limitTen(migrations));
- }
- migrationsFromLastPass.keySet().retainAll(migrations);
- SortedMap<TServerInstance,TabletServerStatus> currentCopy = new TreeMap<>(current);
- Multimap<TServerInstance,String> serverTableIdCopied = HashMultimap.create();
- for (TabletMigration migration : migrationsFromLastPass.values()) {
- TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied,
- migration.tablet.tableId().toString(), migration.oldServer);
- if (fromInfo != null) {
- fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1);
- }
- TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied,
- migration.tablet.tableId().toString(), migration.newServer);
- if (toInfo != null) {
- toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1);
- }
- }
- migrations = EMPTY_MIGRATIONS;
- } else {
- migrationsFromLastPass.clear();
- }
-
- for (String s : tableIdMap.values()) {
- TableId tableId = TableId.of(s);
- String tableName = tableIdToTableName.get(tableId);
- String regexTableName = getPoolNameForTable(tableName);
- SortedMap<TServerInstance,TabletServerStatus> currentView =
- currentGrouped.get(regexTableName);
- if (currentView == null) {
- LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
- continue;
- }
- ArrayList<TabletMigration> newMigrations = new ArrayList<>();
- getBalancerForTable(tableId).balance(currentView, migrations, newMigrations);
-
- if (newMigrations.isEmpty()) {
- tableToTimeSinceNoMigrations.remove(s);
- } else if (tableToTimeSinceNoMigrations.containsKey(s)) {
- if ((now - tableToTimeSinceNoMigrations.get(s)) > HOURS.toMillis(1)) {
- LOG.warn("We have been consistently producing migrations for {}: {}", tableName,
- limitTen(newMigrations));
- }
- } else {
- tableToTimeSinceNoMigrations.put(s, now);
- }
-
- migrationsOut.addAll(newMigrations);
- if (migrationsOut.size() >= myConf.maxTServerMigrations) {
- break;
- }
- }
-
- for (TabletMigration migration : migrationsOut) {
- migrationsFromLastPass.put(migration.tablet, migration);
- }
-
- LOG.info("Migrating tablets for balance: {}", migrationsOut);
- return minBalanceTime;
- }
-
- /**
- * Get a mutable table info for the specified table and server
- */
- private TableInfo getTableInfo(SortedMap<TServerInstance,TabletServerStatus> currentCopy,
- Multimap<TServerInstance,String> serverTableIdCopied, String tableId,
- TServerInstance server) {
- TableInfo newInfo = null;
- if (currentCopy.containsKey(server)) {
- Map<String,TableInfo> newTableMap = currentCopy.get(server).getTableMap();
- if (newTableMap != null) {
- newInfo = newTableMap.get(tableId);
- if (newInfo != null) {
- Collection<String> tableIdCopied = serverTableIdCopied.get(server);
- if (tableIdCopied.isEmpty()) {
- newTableMap = new HashMap<>(newTableMap);
- currentCopy.get(server).setTableMap(newTableMap);
- }
- if (!tableIdCopied.contains(tableId)) {
- newInfo = new TableInfo(newInfo);
- newTableMap.put(tableId, newInfo);
- tableIdCopied.add(tableId);
- }
- }
- }
- }
- return newInfo;
- }
-
- // helper to prepare log messages
- private static String limitTen(Collection<?> iterable) {
- return iterable.stream().limit(10).map(String::valueOf)
- .collect(Collectors.joining(", ", "[", "]"));
- }
-
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/RegexGroupBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/RegexGroupBalancer.java
deleted file mode 100644
index f34608bc79..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/RegexGroupBalancer.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import java.util.Map;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.hadoop.io.Text;
-
-/**
- * A {@link GroupBalancer} that groups tablets using a configurable regex. To use this balancer
- * configure the following settings for your table then configure this balancer for your table.
- *
- * <ul>
- * <li>Set {@code table.custom.balancer.group.regex.pattern} to a regular expression. This regular
- * expression must have one group. The regex is applied to the tablet end row and whatever the regex
- * group matches is used as the group. For example with a regex of {@code (\d\d).*} and an end row
- * of {@code 12abc}, the group for the tablet would be {@code 12}.
- * <li>Set {@code table.custom.balancer.group.regex.default} to a default group. This group is
- * returned for the last tablet in the table and tablets for which the regex does not match.
- * <li>Optionally set {@code table.custom.balancer.group.regex.wait.time} to time (can use time
- * suffixes). This determines how long to wait between balancing. Since this balancer scans the
- * metadata table, may want to set this higher for large tables.
- * </ul>
- *
- * @deprecated since 2.1.0. Use {@link org.apache.accumulo.core.spi.balancer.RegexGroupBalancer}
- * instead.
- */
-@Deprecated(since = "2.1.0")
-public class RegexGroupBalancer extends GroupBalancer {
-
- public static final String REGEX_PROPERTY =
- Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.group.regex.pattern";
- public static final String DEFAUT_GROUP_PROPERTY =
- Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.group.regex.default";
- public static final String WAIT_TIME_PROPERTY =
- Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.group.regex.wait.time";
-
- private final TableId tableId;
-
- public RegexGroupBalancer(TableId tableId) {
- super(tableId);
- this.tableId = tableId;
- }
-
- @Override
- protected long getWaitTime() {
- Map<String,String> customProps = context.getTableConfiguration(tableId)
- .getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
- if (customProps.containsKey(WAIT_TIME_PROPERTY)) {
- return ConfigurationTypeHelper.getTimeInMillis(customProps.get(WAIT_TIME_PROPERTY));
- }
-
- return super.getWaitTime();
- }
-
- @Override
- protected Function<KeyExtent,String> getPartitioner() {
-
- Map<String,String> customProps = context.getTableConfiguration(tableId)
- .getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
- String regex = customProps.get(REGEX_PROPERTY);
- final String defaultGroup = customProps.get(DEFAUT_GROUP_PROPERTY);
-
- final Pattern pattern = Pattern.compile(regex);
-
- return new Function<>() {
-
- @Override
- public String apply(KeyExtent input) {
- Text er = input.endRow();
- if (er == null) {
- return defaultGroup;
- }
-
- Matcher matcher = pattern.matcher(er.toString());
- if (matcher.matches() && matcher.groupCount() == 1) {
- return matcher.group(1);
- }
-
- return defaultGroup;
- }
- };
- }
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
deleted file mode 100644
index 3a42ae0942..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-
-import org.apache.accumulo.core.classloader.ClassLoaderUtil;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.manager.state.tables.TableState;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @deprecated since 2.1.0. Use {@link org.apache.accumulo.core.spi.balancer.TableLoadBalancer}
- * instead.
- */
-@Deprecated(since = "2.1.0")
-public class TableLoadBalancer extends TabletBalancer {
-
- private static final Logger log = LoggerFactory.getLogger(TableLoadBalancer.class);
-
- Map<TableId,TabletBalancer> perTableBalancers = new HashMap<>();
-
- public TableLoadBalancer() {
- log.warn(
- "{} has been deprecated and will be removed in a future release. Please update your "
- + "configuration to use the equivalent {} instead.",
- getClass().getName(),
- org.apache.accumulo.core.spi.balancer.TableLoadBalancer.class.getName());
- }
-
- private TabletBalancer constructNewBalancerForTable(String clazzName, TableId tableId)
- throws Exception {
- String context = null;
- context = ClassLoaderUtil.tableContext(this.context.getTableConfiguration(tableId));
- Class<? extends TabletBalancer> clazz =
- ClassLoaderUtil.loadClass(context, clazzName, TabletBalancer.class);
- Constructor<? extends TabletBalancer> constructor = clazz.getConstructor(TableId.class);
- return constructor.newInstance(tableId);
- }
-
- protected String getLoadBalancerClassNameForTable(TableId table) {
- TableState tableState = context.getTableManager().getTableState(table);
- if (tableState == TableState.ONLINE) {
- return this.context.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
- }
- return null;
- }
-
- protected TabletBalancer getBalancerForTable(TableId tableId) {
- TabletBalancer balancer = perTableBalancers.get(tableId);
-
- String clazzName = getLoadBalancerClassNameForTable(tableId);
-
- if (clazzName == null) {
- clazzName = DefaultLoadBalancer.class.getName();
- }
- if (balancer != null) {
- if (!clazzName.equals(balancer.getClass().getName())) {
- // the balancer class for this table does not match the class specified in the configuration
- try {
- // attempt to construct a balancer with the specified class
- TabletBalancer newBalancer = constructNewBalancerForTable(clazzName, tableId);
- if (newBalancer != null) {
- balancer = newBalancer;
- perTableBalancers.put(tableId, balancer);
- balancer.init(this.context);
- }
-
- log.info("Loaded new class {} for table {}", clazzName, tableId);
- } catch (Exception e) {
- log.warn("Failed to load table balancer class {} for table {}", clazzName, tableId, e);
- }
- }
- }
- if (balancer == null) {
- try {
- balancer = constructNewBalancerForTable(clazzName, tableId);
- log.info("Loaded class {} for table {}", clazzName, tableId);
- } catch (Exception e) {
- log.warn("Failed to load table balancer class {} for table {}", clazzName, tableId, e);
- }
-
- if (balancer == null) {
- log.info("Using balancer {} for table {}", DefaultLoadBalancer.class.getName(), tableId);
- balancer = new DefaultLoadBalancer(tableId);
- }
- perTableBalancers.put(tableId, balancer);
- balancer.init(this.context);
- }
- return balancer;
- }
-
- @Override
- public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
- Map<KeyExtent,TServerInstance> unassigned, Map<KeyExtent,TServerInstance> assignments) {
- // separate the unassigned into tables
- Map<TableId,Map<KeyExtent,TServerInstance>> groupedUnassigned = new HashMap<>();
- unassigned.forEach((ke, lastTserver) -> groupedUnassigned
- .computeIfAbsent(ke.tableId(), k -> new HashMap<>()).put(ke, lastTserver));
- for (Entry<TableId,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet()) {
- Map<KeyExtent,TServerInstance> newAssignments = new HashMap<>();
- getBalancerForTable(e.getKey()).getAssignments(current, e.getValue(), newAssignments);
- assignments.putAll(newAssignments);
- }
- }
-
- private TableOperations tops = null;
-
- protected TableOperations getTableOperations() {
- if (tops == null) {
- tops = this.context.tableOperations();
- }
- return tops;
- }
-
- @Override
- public long balance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
- long minBalanceTime = 5_000;
- // Iterate over the tables and balance each of them
- TableOperations t = getTableOperations();
- if (t == null) {
- return minBalanceTime;
- }
- for (String s : t.tableIdMap().values()) {
- ArrayList<TabletMigration> newMigrations = new ArrayList<>();
- long tableBalanceTime =
- getBalancerForTable(TableId.of(s)).balance(current, migrations, newMigrations);
- if (tableBalanceTime < minBalanceTime) {
- minBalanceTime = tableBalanceTime;
- }
- migrationsOut.addAll(newMigrations);
- }
- return minBalanceTime;
- }
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
deleted file mode 100644
index 6777d93663..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static java.util.stream.Collectors.toList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-
-import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.manager.balancer.AssignmentParamsImpl;
-import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.rpc.ThriftUtil;
-import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.spi.balancer.BalancerEnvironment;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.manager.balancer.BalancerEnvironmentImpl;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.thrift.TException;
-import org.apache.thrift.transport.TTransportException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
-/**
- * This class is responsible for managing the distribution of tablets throughout an Accumulo
- * cluster. In most cases, users will want a balancer implementation which ensures a uniform
- * distribution of tablets, so that no individual tablet server is handling significantly more work
- * than any other.
- *
- * <p>
- * Implementations may wish to store configuration in Accumulo's system configuration using the
- * {@link Property#GENERAL_ARBITRARY_PROP_PREFIX}. They may also benefit from using per-table
- * configuration using {@link Property#TABLE_ARBITRARY_PROP_PREFIX}.
- *
- * @deprecated since 2.1.0. Use {@link org.apache.accumulo.core.spi.balancer.TabletBalancer}
- * instead.
- */
-@Deprecated(since = "2.1.0")
-@SuppressFBWarnings(value = "NM_SAME_SIMPLE_NAME_AS_INTERFACE",
- justification = "Class is deprecated and will be removed.")
-public abstract class TabletBalancer
- implements org.apache.accumulo.core.spi.balancer.TabletBalancer {
-
- private static final Logger log = LoggerFactory.getLogger(TabletBalancer.class);
-
- protected ServerContext context;
-
- @Override
- public void init(BalancerEnvironment balancerEnvironment) {
- var bei = (BalancerEnvironmentImpl) balancerEnvironment;
- init(bei.getContext());
- }
-
- @Override
- public void getAssignments(AssignmentParameters params) {
- AssignmentParamsImpl api = (AssignmentParamsImpl) params;
- getAssignments(api.thriftCurrentStatus(), api.thriftUnassigned(), api.thriftAssignmentsOut());
- }
-
- @Override
- public long balance(BalanceParameters params) {
- BalanceParamsImpl bpi = (BalanceParamsImpl) params;
- List<TabletMigration> migrationsOut = new ArrayList<>();
- long result = balance(bpi.thriftCurrentStatus(), bpi.thriftCurrentMigrations(), migrationsOut);
- migrationsOut.forEach(mo -> bpi.addMigration(mo.tablet, mo.oldServer, mo.newServer));
- return result;
- }
-
- /**
- * Initialize the TabletBalancer. This gives the balancer the opportunity to read the
- * configuration.
- *
- * @deprecated since 2.0.0; use {@link #init(ServerContext)} instead.
- */
- @Deprecated(since = "2.0.0")
- public void init(ServerConfigurationFactory conf) {
- init(conf.getServerContext());
- }
-
- /**
- * Initialize the TabletBalancer. This gives the balancer the opportunity to read the
- * configuration.
- *
- * @since 2.0.0
- */
- public void init(ServerContext context) {
- this.context = context;
- }
-
- /**
- * Assign tablets to tablet servers. This method is called whenever the manager finds tablets that
- * are unassigned.
- *
- * @param current The current table-summary state of all the online tablet servers. Read-only. The
- * TabletServerStatus for each server may be null if the tablet server has not yet
- * responded to a recent request for status.
- * @param unassigned A map from unassigned tablet to the last known tablet server. Read-only.
- * @param assignments A map from tablet to assigned server. Write-only.
- */
- public abstract void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
- Map<KeyExtent,TServerInstance> unassigned, Map<KeyExtent,TServerInstance> assignments);
-
- /**
- * Ask the balancer if any migrations are necessary.
- *
- * If the balancer is going to self-abort due to some environmental constraint (e.g. it requires
- * some minimum number of tservers, or a maximum number of outstanding migrations), it should
- * issue a log message to alert operators. The message should be at WARN normally and at ERROR if
- * the balancer knows that the problem can not self correct. It should not issue these messages
- * more than once a minute.
- *
- * @param current The current table-summary state of all the online tablet servers. Read-only.
- * @param migrations the current set of migrations. Read-only.
- * @param migrationsOut new migrations to perform; should not contain tablets in the current set
- * of migrations. Write-only.
- * @return the time, in milliseconds, to wait before re-balancing.
- *
- * This method will not be called when there are unassigned tablets.
- */
- public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current,
- Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
-
- private static final long ONE_SECOND = 1000L;
- private boolean stuck = false;
- private long stuckNotificationTime = -1L;
-
- protected static final long TIME_BETWEEN_BALANCER_WARNINGS = 60 * ONE_SECOND;
-
- /**
- * A deferred call descendent TabletBalancers use to log why they can't continue. The call is
- * deferred so that TabletBalancer can limit how often messages happen.
- *
- * Implementations should be reused as much as possible.
- *
- * Be sure to pass in a properly scoped Logger instance so that messages indicate what part of the
- * system is having trouble.
- */
- protected abstract static class BalancerProblem implements Runnable {
- protected final Logger balancerLog;
-
- public BalancerProblem(Logger logger) {
- balancerLog = logger;
- }
- }
-
- /**
- * If a TabletBalancer requires active tservers, it should use this problem to indicate when there
- * are none. NoTservers is safe to share with anyone who uses the same Logger. TabletBalancers
- * should have a single static instance.
- */
- protected static class NoTservers extends BalancerProblem {
- public NoTservers(Logger logger) {
- super(logger);
- }
-
- @Override
- public void run() {
- balancerLog.warn("Not balancing because we don't have any tservers");
- }
- }
-
- /**
- * If a TabletBalancer only balances when there are no outstanding migrations, it should use this
- * problem to indicate when they exist.
- *
- * Iff a TabletBalancer makes use of the migrations member to provide samples, then
- * OutstandingMigrations is not thread safe.
- */
- protected static class OutstandingMigrations extends BalancerProblem {
- public Set<KeyExtent> migrations = Collections.emptySet();
-
- public OutstandingMigrations(Logger logger) {
- super(logger);
- }
-
- @Override
- public void run() {
- balancerLog.warn("Not balancing due to {} outstanding migrations.", migrations.size());
- balancerLog.debug("Sample up to 10 outstanding migrations: {}",
- migrations.stream().limit(10).collect(toList()));
- }
- }
-
- /**
- * Warn that a Balancer can't work because of some external restriction. Will not call the
- * provided logging handler more often than TIME_BETWEEN_BALANCER_WARNINGS
- */
- protected void constraintNotMet(BalancerProblem cause) {
- if (stuck) {
- if ((System.currentTimeMillis() - stuckNotificationTime) > TIME_BETWEEN_BALANCER_WARNINGS) {
- cause.run();
- stuckNotificationTime = System.currentTimeMillis();
- }
- } else {
- stuck = true;
- stuckNotificationTime = System.currentTimeMillis();
- }
- }
-
- /**
- * Resets logging about problems meeting an external constraint on balancing.
- */
- protected void resetBalancerErrors() {
- stuck = false;
- }
-
- /**
- * Fetch the tablets for the given table by asking the tablet server. Useful if your balance
- * strategy needs details at the tablet level to decide what tablets to move.
- *
- * @param tserver The tablet server to ask.
- * @param tableId The table id
- * @return a list of tablet statistics
- * @throws ThriftSecurityException tablet server disapproves of your internal System password.
- * @throws TException any other problem
- */
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId)
- throws ThriftSecurityException, TException {
- log.debug("Scanning tablet server {} for table {}", tserver, tableId);
- Client client =
- ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, tserver.getHostAndPort(), context);
- try {
- return client.getTabletStats(TraceUtil.traceInfo(), context.rpcCreds(), tableId.canonical());
- } catch (TTransportException e) {
- log.error("Unable to connect to {}: ", tserver, e);
- } finally {
- ThriftUtil.returnClient(client, context);
- }
- return null;
- }
-
- /**
- * Utility to ensure that the migrations from balance() are consistent:
- * <ul>
- * <li>Tablet objects are not null
- * <li>Source and destination tablet servers are not null and current
- * </ul>
- *
- * @return A list of TabletMigration object that passed sanity checks.
- */
- public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current,
- List<TabletMigration> migrations) {
- List<TabletMigration> result = new ArrayList<>(migrations.size());
- for (TabletMigration m : migrations) {
- if (m.tablet == null) {
- log.warn("Balancer gave back a null tablet {}", m);
- continue;
- }
- if (m.newServer == null) {
- log.warn("Balancer did not set the destination {}", m);
- continue;
- }
- if (m.oldServer == null) {
- log.warn("Balancer did not set the source {}", m);
- continue;
- }
- if (!current.contains(m.oldServer)) {
- log.warn("Balancer wants to move a tablet from a server that is not current: {}", m);
- continue;
- }
- if (!current.contains(m.newServer)) {
- log.warn("Balancer wants to move a tablet to a server that is not current: {}", m);
- continue;
- }
- result.add(m);
- }
- return result;
- }
-
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletMigration.java b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletMigration.java
deleted file mode 100644
index 132e22d1f3..0000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletMigration.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.state;
-
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TServerInstance;
-
-/**
- * @deprecated since 2.1.0. Use balancers in org.apache.accumulo.core.spi.balancer instead.
- */
-@Deprecated(since = "2.1.0")
-public class TabletMigration {
- public KeyExtent tablet;
- public TServerInstance oldServer;
- public TServerInstance newServer;
-
- public TabletMigration(KeyExtent extent, TServerInstance before, TServerInstance after) {
- this.tablet = extent;
- this.oldServer = before;
- this.newServer = after;
- }
-
- @Override
- public String toString() {
- return tablet + ": " + oldServer + " -> " + newServer;
- }
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
deleted file mode 100644
index 7d935d28d9..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.function.Predicate;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.clientImpl.Namespace;
-import org.apache.accumulo.core.clientImpl.TableOperationsImpl;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.ConfigurationCopy;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
-import org.apache.accumulo.core.data.InstanceId;
-import org.apache.accumulo.core.data.NamespaceId;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.conf.NamespaceConfiguration;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.conf.codec.VersionedProperties;
-import org.apache.accumulo.server.conf.store.NamespacePropKey;
-import org.apache.accumulo.server.conf.store.PropStore;
-import org.apache.accumulo.server.conf.store.TablePropKey;
-import org.apache.accumulo.server.conf.store.impl.ZooPropStore;
-import org.apache.hadoop.io.Text;
-import org.easymock.EasyMock;
-
-@Deprecated(since = "2.1.0")
-public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
-
- protected static class TestTable {
- private String tableName;
- private TableId id;
-
- TestTable(String tableName, TableId id) {
- this.tableName = tableName;
- this.id = id;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public TableId getId() {
- return id;
- }
- }
-
- protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>();
- {
- DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "7s");
- DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY,
- "4");
- DEFAULT_TABLE_PROPERTIES
- .put(HostRegexTableLoadBalancer.HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY, "10");
- DEFAULT_TABLE_PROPERTIES
- .put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r01.*");
- DEFAULT_TABLE_PROPERTIES
- .put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r02.*");
- DEFAULT_TABLE_PROPERTIES.put(Property.TABLE_LOAD_BALANCER.getKey(),
- TestDefaultBalancer.class.getName());
- }
-
- private static SiteConfiguration siteConfg = SiteConfiguration.empty().build();
-
- protected static class TestServerConfigurationFactory extends ServerConfigurationFactory {
-
- final ServerContext context;
- private ConfigurationCopy config;
-
- public TestServerConfigurationFactory(ServerContext context) {
- super(context, siteConfg);
- this.context = context;
- this.config = new ConfigurationCopy(DEFAULT_TABLE_PROPERTIES);
- }
-
- @Override
- public synchronized AccumuloConfiguration getSystemConfiguration() {
- return config;
- }
-
- @Override
- public TableConfiguration getTableConfiguration(final TableId tableId) {
- // create a dummy namespaceConfiguration to satisfy requireNonNull in TableConfiguration
- // constructor
- NamespaceConfiguration dummyConf = new NamespaceConfiguration(context, Namespace.DEFAULT.id(),
- DefaultConfiguration.getInstance());
- return new TableConfiguration(context, tableId, dummyConf) {
- @Override
- public String get(Property property) {
- return getSystemConfiguration().get(property.name());
- }
-
- @Override
- public void getProperties(Map<String,String> props, Predicate<String> filter) {
- getSystemConfiguration().getProperties(props, filter);
- }
-
- @Override
- public long getUpdateCount() {
- return 0;
- }
- };
- }
- }
-
- protected static final TestTable FOO = new TestTable("foo", TableId.of("1"));
- protected static final TestTable BAR = new TestTable("bar", TableId.of("2"));
- protected static final TestTable BAZ = new TestTable("baz", TableId.of("3"));
-
- protected class TestDefaultBalancer extends DefaultLoadBalancer {
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId) {
- String tableName = idToTableName(tableId);
- TServerInstance initialLocation = initialTableLocation.get(tableName);
- if (tserver.equals(initialLocation)) {
- List<TabletStats> list = new ArrayList<>(5);
- for (KeyExtent extent : tableExtents.get(tableName)) {
- TabletStats stats = new TabletStats();
- stats.setExtent(extent.toThrift());
- list.add(stats);
- }
- return list;
- }
- return null;
- }
- }
-
- protected ServerContext createMockContext() {
- InstanceId instanceId = InstanceId.of(UUID.randomUUID());
-
- ServerContext mockContext = createMock(ServerContext.class);
- PropStore propStore = createMock(ZooPropStore.class);
- expect(mockContext.getProperties()).andReturn(new Properties()).anyTimes();
- expect(mockContext.getZooKeepers()).andReturn("").anyTimes();
- expect(mockContext.getInstanceName()).andReturn("test").anyTimes();
- expect(mockContext.getZooKeepersSessionTimeOut()).andReturn(30).anyTimes();
- expect(mockContext.getInstanceID()).andReturn(instanceId).anyTimes();
- expect(mockContext.getZooKeeperRoot()).andReturn(Constants.ZROOT + "/1111").anyTimes();
-
- expect(mockContext.getPropStore()).andReturn(propStore).anyTimes();
- propStore.registerAsListener(anyObject(), anyObject());
- expectLastCall().anyTimes();
-
- expect(propStore.get(eq(NamespacePropKey.of(instanceId, NamespaceId.of("+default")))))
- .andReturn(new VersionedProperties()).anyTimes();
-
- expect(propStore.get(eq(TablePropKey.of(instanceId, TableId.of("1")))))
- .andReturn(new VersionedProperties()).anyTimes();
-
- expect(propStore.get(eq(TablePropKey.of(instanceId, TableId.of("2")))))
- .andReturn(new VersionedProperties()).anyTimes();
-
- expect(propStore.get(eq(TablePropKey.of(instanceId, TableId.of("3")))))
- .andReturn(new VersionedProperties()).anyTimes();
-
- replay(propStore);
- return mockContext;
- }
-
- protected final Map<String,String> servers = new HashMap<>(15);
- protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers = new TreeMap<>();
- protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
- protected final Map<String,TServerInstance> initialTableLocation = new HashMap<>(3);
-
- {
- servers.put("192.168.0.1", "r01s01");
- servers.put("192.168.0.2", "r01s02");
- servers.put("192.168.0.3", "r01s03");
- servers.put("192.168.0.4", "r01s04");
- servers.put("192.168.0.5", "r01s05");
- servers.put("192.168.0.6", "r02s01");
- servers.put("192.168.0.7", "r02s02");
- servers.put("192.168.0.8", "r02s03");
- servers.put("192.168.0.9", "r02s04");
- servers.put("192.168.0.10", "r02s05");
- servers.put("192.168.0.11", "r03s01");
- servers.put("192.168.0.12", "r03s02");
- servers.put("192.168.0.13", "r03s03");
- servers.put("192.168.0.14", "r03s04");
- servers.put("192.168.0.15", "r03s05");
-
- allTabletServers.put(new TServerInstance("192.168.0.1:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.2:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.3:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.4:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.5:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.6:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.7:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.8:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.9:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.10:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.11:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.12:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.13:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
-
- initialTableLocation.put(FOO.getTableName(), new TServerInstance("192.168.0.1:9997", 1));
- initialTableLocation.put(BAR.getTableName(), new TServerInstance("192.168.0.6:9997", 1));
- initialTableLocation.put(BAZ.getTableName(), new TServerInstance("192.168.0.11:9997", 1));
-
- tableExtents.put(FOO.getTableName(), new ArrayList<>());
- tableExtents.get(FOO.getTableName())
- .add(new KeyExtent(FOO.getId(), new Text("1"), new Text("0")));
- tableExtents.get(FOO.getTableName())
- .add(new KeyExtent(FOO.getId(), new Text("2"), new Text("1")));
- tableExtents.get(FOO.getTableName())
- .add(new KeyExtent(FOO.getId(), new Text("3"), new Text("2")));
- tableExtents.get(FOO.getTableName())
- .add(new KeyExtent(FOO.getId(), new Text("4"), new Text("3")));
- tableExtents.get(FOO.getTableName())
- .add(new KeyExtent(FOO.getId(), new Text("5"), new Text("4")));
- tableExtents.put(BAR.getTableName(), new ArrayList<>());
- tableExtents.get(BAR.getTableName())
- .add(new KeyExtent(BAR.getId(), new Text("11"), new Text("10")));
- tableExtents.get(BAR.getTableName())
- .add(new KeyExtent(BAR.getId(), new Text("12"), new Text("11")));
- tableExtents.get(BAR.getTableName())
- .add(new KeyExtent(BAR.getId(), new Text("13"), new Text("12")));
- tableExtents.get(BAR.getTableName())
- .add(new KeyExtent(BAR.getId(), new Text("14"), new Text("13")));
- tableExtents.get(BAR.getTableName())
- .add(new KeyExtent(BAR.getId(), new Text("15"), new Text("14")));
- tableExtents.put(BAZ.getTableName(), new ArrayList<>());
- tableExtents.get(BAZ.getTableName())
- .add(new KeyExtent(BAZ.getId(), new Text("21"), new Text("20")));
- tableExtents.get(BAZ.getTableName())
- .add(new KeyExtent(BAZ.getId(), new Text("22"), new Text("21")));
- tableExtents.get(BAZ.getTableName())
- .add(new KeyExtent(BAZ.getId(), new Text("23"), new Text("22")));
- tableExtents.get(BAZ.getTableName())
- .add(new KeyExtent(BAZ.getId(), new Text("24"), new Text("23")));
- tableExtents.get(BAZ.getTableName())
- .add(new KeyExtent(BAZ.getId(), new Text("25"), new Text("24")));
-
- }
-
- protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
- String tid = ke.tableId().canonical();
- String host = tsi.getHost();
- if (tid.equals("1")
- && (host.equals("192.168.0.1") || host.equals("192.168.0.2") || host.equals("192.168.0.3")
- || host.equals("192.168.0.4") || host.equals("192.168.0.5"))) {
- return true;
- } else if (tid.equals("2")
- && (host.equals("192.168.0.6") || host.equals("192.168.0.7") || host.equals("192.168.0.8")
- || host.equals("192.168.0.9") || host.equals("192.168.0.10"))) {
- return true;
- } else {
- return tid.equals("3") && (host.equals("192.168.0.11") || host.equals("192.168.0.12")
- || host.equals("192.168.0.13") || host.equals("192.168.0.14")
- || host.equals("192.168.0.15"));
- }
- }
-
- protected String idToTableName(TableId id) {
- if (id.equals(FOO.getId())) {
- return FOO.getTableName();
- } else if (id.equals(BAR.getId())) {
- return BAR.getTableName();
- } else if (id.equals(BAZ.getId())) {
- return BAZ.getTableName();
- } else {
- return null;
- }
- }
-
- @Override
- protected TableOperations getTableOperations() {
- return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
- @Override
- public Map<String,String> tableIdMap() {
- HashMap<String,String> tables = new HashMap<>();
- tables.put(FOO.getTableName(), FOO.getId().canonical());
- tables.put(BAR.getTableName(), BAR.getId().canonical());
- tables.put(BAZ.getTableName(), BAZ.getId().canonical());
- return tables;
- }
-
- @Override
- public SortedSet<String> list() {
- TreeSet<String> tables = new TreeSet<>();
- tables.add(BAR.getTableName());
- tables.add(BAZ.getTableName());
- tables.add(FOO.getTableName());
- return tables;
- }
- };
- }
-
- @Override
- protected TabletBalancer getBalancerForTable(TableId table) {
- return new TestDefaultBalancer();
- }
-
- @Override
- protected String getNameFromIp(String hostIp) throws UnknownHostException {
- if (servers.containsKey(hostIp)) {
- return servers.get(hostIp);
- } else {
- throw new UnknownHostException();
- }
- }
-
- protected SortedMap<TServerInstance,TabletServerStatus> createCurrent(int numTservers) {
- String base = "192.168.0.";
- TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
- for (int i = 1; i <= numTservers; i++) {
- TabletServerStatus status = new TabletServerStatus();
- Map<String,TableInfo> tableMap = new HashMap<>();
- tableMap.put(FOO.getId().canonical(), new TableInfo());
- tableMap.put(BAR.getId().canonical(), new TableInfo());
- tableMap.put(BAZ.getId().canonical(), new TableInfo());
- status.setTableMap(tableMap);
- current.put(new TServerInstance(base + i + ":9997", 1), status);
- }
- // now put all of the tablets on one server
- for (Map.Entry<String,TServerInstance> entry : initialTableLocation.entrySet()) {
- TabletServerStatus status = current.get(entry.getValue());
- if (status != null) {
- String tableId = getTableOperations().tableIdMap().get(entry.getKey());
- status.getTableMap().get(tableId).setOnlineTablets(5);
- }
- }
- return current;
- }
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
deleted file mode 100644
index 02835e2a63..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.hadoop.io.Text;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-@Deprecated(since = "2.1.0")
-public class DefaultLoadBalancerTest {
-
- class FakeTServer {
- List<KeyExtent> extents = new ArrayList<>();
-
- TabletServerStatus getStatus() {
- TabletServerStatus result = new TabletServerStatus();
- result.tableMap = new HashMap<>();
- for (KeyExtent extent : extents) {
- TableId tableId = extent.tableId();
- TableInfo info = result.tableMap.get(tableId.canonical());
- if (info == null) {
- result.tableMap.put(tableId.canonical(), info = new TableInfo());
- }
- info.onlineTablets++;
- info.recs = info.onlineTablets;
- info.ingestRate = 123.;
- info.queryRate = 456.;
- }
- return result;
- }
- }
-
- Map<TServerInstance,FakeTServer> servers = new HashMap<>();
- Map<KeyExtent,TServerInstance> last = new HashMap<>();
-
- class TestDefaultLoadBalancer extends DefaultLoadBalancer {
-
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId table) {
- List<TabletStats> result = new ArrayList<>();
- for (KeyExtent extent : servers.get(tserver).extents) {
- if (extent.tableId().equals(table)) {
- result.add(new TabletStats(extent.toThrift(), null, null, null, 0L, 0., 0., 0));
- }
- }
- return result;
- }
- }
-
- @BeforeEach
- public void setUp() {
- last.clear();
- servers.clear();
- }
-
- @Test
- public void testAssignMigrations() {
- servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), "a"),
- new FakeTServer());
- servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.2", 1234), "b"),
- new FakeTServer());
- servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.3", 1234), "c"),
- new FakeTServer());
- List<KeyExtent> metadataTable = new ArrayList<>();
- String table = "t1";
- metadataTable.add(makeExtent(table, null, null));
- table = "t2";
- metadataTable.add(makeExtent(table, "a", null));
- metadataTable.add(makeExtent(table, null, "a"));
- table = "t3";
- metadataTable.add(makeExtent(table, "a", null));
- metadataTable.add(makeExtent(table, "b", "a"));
- metadataTable.add(makeExtent(table, "c", "b"));
- metadataTable.add(makeExtent(table, "d", "c"));
- metadataTable.add(makeExtent(table, "e", "d"));
- metadataTable.add(makeExtent(table, null, "e"));
- Collections.sort(metadataTable);
-
- TestDefaultLoadBalancer balancer = new TestDefaultLoadBalancer();
-
- SortedMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
- for (Entry<TServerInstance,FakeTServer> entry : servers.entrySet()) {
- current.put(entry.getKey(), entry.getValue().getStatus());
- }
- assignTablets(metadataTable, servers, current, balancer);
-
- // Verify that the counts on the tables are correct
- Map<String,Integer> expectedCounts = new HashMap<>();
- expectedCounts.put("t1", 1);
- expectedCounts.put("t2", 1);
- expectedCounts.put("t3", 2);
- checkBalance(metadataTable, servers, expectedCounts);
-
- // Rebalance once
- for (Entry<TServerInstance,FakeTServer> entry : servers.entrySet()) {
- current.put(entry.getKey(), entry.getValue().getStatus());
- }
-
- // Nothing should happen, we are balanced
- ArrayList<TabletMigration> out = new ArrayList<>();
- balancer.getMigrations(current, out);
- assertEquals(out.size(), 0);
-
- // Take down a tabletServer
- TServerInstance first = current.keySet().iterator().next();
- current.remove(first);
- FakeTServer remove = servers.remove(first);
-
- // reassign offline extents
- assignTablets(remove.extents, servers, current, balancer);
- checkBalance(metadataTable, servers, null);
- }
-
- private void assignTablets(List<KeyExtent> metadataTable,
- Map<TServerInstance,FakeTServer> servers,
- SortedMap<TServerInstance,TabletServerStatus> status, TestDefaultLoadBalancer balancer) {
- // Assign tablets
- for (KeyExtent extent : metadataTable) {
- TServerInstance assignment = balancer.getAssignment(status, last.get(extent));
- assertNotNull(assignment);
- assertFalse(servers.get(assignment).extents.contains(extent));
- servers.get(assignment).extents.add(extent);
- last.put(extent, assignment);
- }
- }
-
- SortedMap<TServerInstance,TabletServerStatus>
- getAssignments(Map<TServerInstance,FakeTServer> servers) {
- SortedMap<TServerInstance,TabletServerStatus> result = new TreeMap<>();
- for (Entry<TServerInstance,FakeTServer> entry : servers.entrySet()) {
- result.put(entry.getKey(), entry.getValue().getStatus());
- }
- return result;
- }
-
- @Test
- public void testUnevenAssignment() {
- for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
- String cString = Character.toString(c);
- HostAndPort fakeAddress = HostAndPort.fromParts("127.0.0.1", c);
- TServerInstance tsi = new TServerInstance(fakeAddress, cString);
- FakeTServer fakeTServer = new FakeTServer();
- servers.put(tsi, fakeTServer);
- fakeTServer.extents.add(makeExtent(cString, null, null));
- }
- // Put more tablets on one server, but not more than the number of servers
- Entry<TServerInstance,FakeTServer> first = servers.entrySet().iterator().next();
- first.getValue().extents.add(makeExtent("newTable", "a", null));
- first.getValue().extents.add(makeExtent("newTable", "b", "a"));
- first.getValue().extents.add(makeExtent("newTable", "c", "b"));
- first.getValue().extents.add(makeExtent("newTable", "d", "c"));
- first.getValue().extents.add(makeExtent("newTable", "e", "d"));
- first.getValue().extents.add(makeExtent("newTable", "f", "e"));
- first.getValue().extents.add(makeExtent("newTable", "g", "f"));
- first.getValue().extents.add(makeExtent("newTable", "h", "g"));
- first.getValue().extents.add(makeExtent("newTable", "i", null));
- TestDefaultLoadBalancer balancer = new TestDefaultLoadBalancer();
- Set<KeyExtent> migrations = Collections.emptySet();
- int moved = 0;
- // balance until we can't balance no more!
- while (true) {
- List<TabletMigration> migrationsOut = new ArrayList<>();
- balancer.balance(getAssignments(servers), migrations, migrationsOut);
- if (migrationsOut.isEmpty()) {
- break;
- }
- for (TabletMigration migration : migrationsOut) {
- if (servers.get(migration.oldServer).extents.remove(migration.tablet)) {
- moved++;
- }
- servers.get(migration.newServer).extents.add(migration.tablet);
- }
- }
- assertEquals(8, moved);
- }
-
- @Test
- public void testUnevenAssignment2() {
- // make 26 servers
- for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
- String cString = Character.toString(c);
- HostAndPort fakeAddress = HostAndPort.fromParts("127.0.0.1", c);
- TServerInstance tsi = new TServerInstance(fakeAddress, cString);
- FakeTServer fakeTServer = new FakeTServer();
- servers.put(tsi, fakeTServer);
- }
- // put 60 tablets on 25 of them
- List<Entry<TServerInstance,FakeTServer>> shortList = new ArrayList<>(servers.entrySet());
- Entry<TServerInstance,FakeTServer> shortServer = shortList.remove(0);
- int c = 0;
- for (int i = 0; i < 60; i++) {
- for (Entry<TServerInstance,FakeTServer> entry : shortList) {
- entry.getValue().extents.add(makeExtent("t" + c, null, null));
- }
- }
- // put 10 on the that short server:
- for (int i = 0; i < 10; i++) {
- shortServer.getValue().extents.add(makeExtent("s" + i, null, null));
- }
-
- TestDefaultLoadBalancer balancer = new TestDefaultLoadBalancer();
- Set<KeyExtent> migrations = Collections.emptySet();
- int moved = 0;
- // balance until we can't balance no more!
- while (true) {
- List<TabletMigration> migrationsOut = new ArrayList<>();
- balancer.balance(getAssignments(servers), migrations, migrationsOut);
- if (migrationsOut.isEmpty()) {
- break;
- }
- for (TabletMigration migration : migrationsOut) {
- if (servers.get(migration.oldServer).extents.remove(migration.tablet)) {
- moved++;
- }
- last.remove(migration.tablet);
- servers.get(migration.newServer).extents.add(migration.tablet);
- last.put(migration.tablet, migration.newServer);
- }
- }
- // average is 58, with 2 at 59: we need 48 more moved to the short server
- assertEquals(48, moved);
- }
-
- private void checkBalance(List<KeyExtent> metadataTable, Map<TServerInstance,FakeTServer> servers,
- Map<String,Integer> expectedCounts) {
- // Verify they are spread evenly over the cluster
- int average = metadataTable.size() / servers.size();
- for (FakeTServer server : servers.values()) {
- int diff = server.extents.size() - average;
- if (diff < 0) {
- fail("average number of tablets is " + average + " but a server has "
- + server.extents.size());
- }
- if (diff > 1) {
- fail("average number of tablets is " + average + " but a server has "
- + server.extents.size());
- }
- }
-
- if (expectedCounts != null) {
- for (FakeTServer server : servers.values()) {
- Map<String,Integer> counts = new HashMap<>();
- server.extents.forEach(extent -> {
- String t = extent.tableId().canonical();
- counts.putIfAbsent(t, 0);
- counts.put(t, counts.get(t) + 1);
- });
- counts.forEach((k, v) -> assertEquals(expectedCounts.get(k), v));
- }
- }
- }
-
- private static KeyExtent makeExtent(String table, String end, String prev) {
- return new KeyExtent(TableId.of(table), toText(end), toText(prev));
- }
-
- private static Text toText(String value) {
- if (value != null) {
- return new Text(value);
- }
- return null;
- }
-
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/GroupBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/GroupBalancerTest.java
deleted file mode 100644
index a25bfcf798..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/GroupBalancerTest.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.function.Function;
-
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.util.MapCounter;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.hadoop.io.Text;
-import org.junit.jupiter.api.Test;
-
-@Deprecated(since = "2.1.0")
-public class GroupBalancerTest {
-
- private static final SecureRandom random = new SecureRandom();
-
- private static Function<KeyExtent,String> partitioner = new Function<>() {
-
- @Override
- public String apply(KeyExtent input) {
- return (input == null || input.endRow() == null) ? null
- : input.endRow().toString().substring(0, 2);
- }
- };
-
- public static class TabletServers {
- private final Set<TServerInstance> tservers = new HashSet<>();
- private final Map<KeyExtent,TServerInstance> tabletLocs = new HashMap<>();
-
- public void addTservers(String... locs) {
- for (String loc : locs) {
- addTserver(loc);
- }
- }
-
- public void addTserver(String loc) {
- tservers.add(new TServerInstance(loc, 6));
- }
-
- public void addTablet(String er, String location) {
- TServerInstance tsi = new TServerInstance(location, 6);
- tabletLocs.put(new KeyExtent(TableId.of("b"), er == null ? null : new Text(er), null),
- new TServerInstance(location, 6));
- tservers.add(tsi);
- }
-
- public void balance() {
- balance(10000);
- }
-
- public void balance(final int maxMigrations) {
- GroupBalancer balancer = new GroupBalancer(TableId.of("1")) {
-
- @Override
- protected Map<KeyExtent,TServerInstance> getLocationProvider() {
- return tabletLocs;
- }
-
- @Override
- protected Function<KeyExtent,String> getPartitioner() {
- return partitioner;
- }
-
- @Override
- protected long getWaitTime() {
- return 0;
- }
-
- @Override
- protected int getMaxMigrations() {
- return maxMigrations;
- }
- };
-
- balance(balancer, maxMigrations);
- }
-
- public void balance(TabletBalancer balancer, int maxMigrations) {
-
- while (true) {
- Set<KeyExtent> migrations = new HashSet<>();
- List<TabletMigration> migrationsOut = new ArrayList<>();
- SortedMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
-
- for (TServerInstance tsi : tservers) {
- current.put(tsi, new TabletServerStatus());
- }
-
- balancer.balance(current, migrations, migrationsOut);
-
- assertTrue(migrationsOut.size() <= (maxMigrations + 5),
- "Max Migration exceeded " + maxMigrations + " " + migrationsOut.size());
-
- for (TabletMigration tabletMigration : migrationsOut) {
- assertEquals(tabletLocs.get(tabletMigration.tablet), tabletMigration.oldServer);
- assertTrue(tservers.contains(tabletMigration.newServer));
-
- tabletLocs.put(tabletMigration.tablet, tabletMigration.newServer);
- }
-
- if (migrationsOut.isEmpty()) {
- break;
- }
- }
-
- checkBalance();
- }
-
- void checkBalance() {
- MapCounter<String> groupCounts = new MapCounter<>();
- Map<TServerInstance,MapCounter<String>> tserverGroupCounts = new HashMap<>();
-
- for (Entry<KeyExtent,TServerInstance> entry : tabletLocs.entrySet()) {
- String group = partitioner.apply(entry.getKey());
- TServerInstance loc = entry.getValue();
-
- groupCounts.increment(group, 1);
- MapCounter<String> tgc = tserverGroupCounts.get(loc);
- if (tgc == null) {
- tgc = new MapCounter<>();
- tserverGroupCounts.put(loc, tgc);
- }
-
- tgc.increment(group, 1);
- }
-
- Map<String,Integer> expectedCounts = new HashMap<>();
-
- int totalExtra = 0;
- for (String group : groupCounts.keySet()) {
- long groupCount = groupCounts.get(group);
- totalExtra += groupCount % tservers.size();
- expectedCounts.put(group, (int) (groupCount / tservers.size()));
- }
-
- // The number of extra tablets from all groups that each tserver must have.
- int expectedExtra = totalExtra / tservers.size();
- int maxExtraGroups = expectedExtra + ((totalExtra % tservers.size() > 0) ? 1 : 0);
-
- for (Entry<TServerInstance,MapCounter<String>> entry : tserverGroupCounts.entrySet()) {
- MapCounter<String> tgc = entry.getValue();
- int tserverExtra = 0;
- for (String group : groupCounts.keySet()) {
- assertTrue(tgc.get(group) >= expectedCounts.get(group));
- assertTrue(tgc.get(group) <= expectedCounts.get(group) + 1,
- "Group counts not as expected group:" + group + " actual:" + tgc.get(group)
- + " expected:" + (expectedCounts.get(group) + 1) + " tserver:" + entry.getKey());
- tserverExtra += tgc.get(group) - expectedCounts.get(group);
- }
-
- assertTrue(tserverExtra >= expectedExtra);
- assertTrue(tserverExtra <= maxExtraGroups);
- }
- }
- }
-
- @Test
- public void testSingleGroup() {
-
- String[][] tests = {new String[] {"a", "b", "c", "d"}, new String[] {"a", "b", "c"},
- new String[] {"a", "b", "c", "d", "e"}, new String[] {"a", "b", "c", "d", "e", "f", "g"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, new String[] {"a"}};
-
- for (String[] suffixes : tests) {
- for (int maxTS = 1; maxTS <= 4; maxTS++) {
- TabletServers tservers = new TabletServers();
- tservers = new TabletServers();
- int ts = 0;
- for (String s : suffixes) {
- tservers.addTablet("01" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- tservers.addTservers("192.168.1.2:9997", "192.168.1.3:9997", "192.168.1.4:9997");
- tservers.balance();
- tservers.balance();
- }
- }
- }
-
- @Test
- public void testTwoGroups() {
- String[][] tests = {new String[] {"a", "b", "c", "d"}, new String[] {"a", "b", "c"},
- new String[] {"a", "b", "c", "d", "e"}, new String[] {"a", "b", "c", "d", "e", "f", "g"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, new String[] {"a"}};
-
- for (String[] suffixes1 : tests) {
- for (String[] suffixes2 : tests) {
- for (int maxTS = 1; maxTS <= 4; maxTS++) {
- TabletServers tservers = new TabletServers();
- tservers = new TabletServers();
- int ts = 0;
- for (String s : suffixes1) {
- tservers.addTablet("01" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- for (String s : suffixes2) {
- tservers.addTablet("02" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- tservers.addTservers("192.168.1.2:9997", "192.168.1.3:9997", "192.168.1.4:9997");
- tservers.balance();
- tservers.balance();
- }
- }
- }
- }
-
- @Test
- public void testThreeGroups() {
- String[][] tests = {new String[] {"a", "b", "c", "d"}, new String[] {"a", "b", "c"},
- new String[] {"a", "b", "c", "d", "e"}, new String[] {"a", "b", "c", "d", "e", "f", "g"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h"},
- new String[] {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, new String[] {"a"}};
-
- for (String[] suffixes1 : tests) {
- for (String[] suffixes2 : tests) {
- for (String[] suffixes3 : tests) {
- for (int maxTS = 1; maxTS <= 4; maxTS++) {
- TabletServers tservers = new TabletServers();
- tservers = new TabletServers();
- int ts = 0;
- for (String s : suffixes1) {
- tservers.addTablet("01" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- for (String s : suffixes2) {
- tservers.addTablet("02" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- for (String s : suffixes3) {
- tservers.addTablet("03" + s, "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- tservers.addTservers("192.168.1.2:9997", "192.168.1.3:9997", "192.168.1.4:9997");
- tservers.balance();
- tservers.balance();
- }
- }
- }
- }
- }
-
- @Test
- public void testManySingleTabletGroups() {
-
- for (int numGroups = 1; numGroups <= 13; numGroups++) {
- for (int maxTS = 1; maxTS <= 4; maxTS++) {
- TabletServers tservers = new TabletServers();
- tservers = new TabletServers();
- int ts = 0;
-
- for (int group = 1; group <= numGroups; group++) {
- tservers.addTablet(String.format("%02d:p", group),
- "192.168.1." + ((ts++ % maxTS) + 1) + ":9997");
- }
-
- tservers.addTservers("192.168.1.2:9997", "192.168.1.3:9997", "192.168.1.4:9997");
-
- tservers.balance();
- tservers.balance();
- }
- }
- }
-
- @Test
- public void testMaxMigrations() {
-
- for (int max : new int[] {1, 2, 3, 7, 10, 30}) {
- TabletServers tservers = new TabletServers();
-
- for (int i = 1; i <= 9; i++) {
- tservers.addTablet("01" + i, "192.168.1.1:9997");
- }
-
- for (int i = 1; i <= 4; i++) {
- tservers.addTablet("02" + i, "192.168.1.2:9997");
- }
-
- for (int i = 1; i <= 5; i++) {
- tservers.addTablet("03" + i, "192.168.1.3:9997");
- }
-
- tservers.addTservers("192.168.1.4:9997", "192.168.1.5:9997");
-
- tservers.balance(max);
- }
- }
-
- @Test
- public void bigTest() {
- TabletServers tservers = new TabletServers();
-
- for (int g = 1; g <= 60; g++) {
- for (int t = 1; t <= 241; t++) {
- tservers.addTablet(String.format("%02d:%d", g, t),
- "192.168.1." + (random.nextInt(249) + 1) + ":9997");
- }
- }
-
- for (int i = 1; i <= 250; i++) {
- tservers.addTserver("192.168.1." + i + ":9997");
- }
-
- tservers.balance(1000);
- }
-
- @Test
- public void bigTest2() {
- TabletServers tservers = new TabletServers();
-
- for (int g = 1; g <= 60; g++) {
- for (int t = 1; t <= random.nextInt(1000); t++) {
- tservers.addTablet(String.format("%02d:%d", g, t),
- "192.168.1." + (random.nextInt(249) + 1) + ":9997");
- }
- }
-
- for (int i = 1; i <= 250; i++) {
- tservers.addTserver("192.168.1." + i + ":9997");
- }
-
- tservers.balance(1000);
- }
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
deleted file mode 100644
index 186ee5f265..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.conf.ConfigurationCopy;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.junit.jupiter.api.Test;
-
-@Deprecated(since = "2.1.0")
-public class HostRegexTableLoadBalancerReconfigurationTest
- extends BaseHostRegexTableLoadBalancerTest {
-
- private Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
-
- @Test
- public void testConfigurationChanges() {
- ServerContext context1 = createMockContext();
- replay(context1);
- final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(context1);
- ServerContext context2 = createMockContext();
- expect(context2.getConfiguration()).andReturn(factory.getSystemConfiguration()).anyTimes();
- expect(context2.getTableConfiguration(FOO.getId()))
- .andReturn(factory.getTableConfiguration(FOO.getId())).anyTimes();
- expect(context2.getTableConfiguration(BAR.getId()))
- .andReturn(factory.getTableConfiguration(BAR.getId())).anyTimes();
- expect(context2.getTableConfiguration(BAZ.getId()))
- .andReturn(factory.getTableConfiguration(BAZ.getId())).anyTimes();
- replay(context2);
- init(context2);
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- for (List<KeyExtent> extents : tableExtents.values()) {
- for (KeyExtent ke : extents) {
- unassigned.put(ke, null);
- }
- }
- this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(15, assignments.size());
- // Ensure unique tservers
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
- if (e.getKey().equals(e2.getKey())) {
- continue;
- }
- if (e.getValue().equals(e2.getValue())) {
- fail("Assignment failure. " + e.getKey() + " and " + e2.getKey()
- + " are assigned to the same host: " + e.getValue());
- }
- }
- }
- // Ensure assignments are correct
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- if (!tabletInBounds(e.getKey(), e.getValue())) {
- fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
- }
- }
- Set<KeyExtent> migrations = new HashSet<>();
- List<TabletMigration> migrationsOut = new ArrayList<>();
- // Wait to trigger the out of bounds check which will call our version of
- // getOnlineTabletsForTable
- UtilWaitThread.sleep(3000);
- this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
- assertEquals(0, migrationsOut.size());
- // Change property, simulate call by TableConfWatcher
-
- ((ConfigurationCopy) factory.getSystemConfiguration())
- .set(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
-
- // Wait to trigger the out of bounds check and the repool check
- UtilWaitThread.sleep(10000);
- this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
- assertEquals(5, migrationsOut.size());
- for (TabletMigration migration : migrationsOut) {
- assertTrue(migration.newServer.getHost().startsWith("192.168.0.1")
- || migration.newServer.getHost().startsWith("192.168.0.2")
- || migration.newServer.getHost().startsWith("192.168.0.3")
- || migration.newServer.getHost().startsWith("192.168.0.4")
- || migration.newServer.getHost().startsWith("192.168.0.5"));
- }
- }
-
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId) {
- List<TabletStats> tablets = new ArrayList<>();
- // Report assignment information
- for (Entry<KeyExtent,TServerInstance> e : this.assignments.entrySet()) {
- if (e.getValue().equals(tserver) && e.getKey().tableId().equals(tableId)) {
- TabletStats ts = new TabletStats();
- ts.setExtent(e.getKey().toThrift());
- tablets.add(ts);
- }
- }
- return tablets;
- }
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
deleted file mode 100644
index 8a4071668b..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ /dev/null
@@ -1,480 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.ConfigurationCopy;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.junit.jupiter.api.Test;
-
-@Deprecated(since = "2.1.0")
-public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalancerTest {
-
- public void init() {
- ServerContext context1 = createMockContext();
- replay(context1);
- final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(context1);
- initFactory(factory);
- }
-
- private void initFactory(ServerConfigurationFactory factory) {
- ServerContext context = createMockContext();
- expect(context.getConfiguration()).andReturn(factory.getSystemConfiguration()).anyTimes();
- expect(context.getTableConfiguration(FOO.getId()))
- .andReturn(factory.getTableConfiguration(FOO.getId())).anyTimes();
- expect(context.getTableConfiguration(BAR.getId()))
- .andReturn(factory.getTableConfiguration(BAR.getId())).anyTimes();
- expect(context.getTableConfiguration(BAZ.getId()))
- .andReturn(factory.getTableConfiguration(BAZ.getId())).anyTimes();
- replay(context);
- init(context);
- }
-
- @Test
- public void testInit() {
- init();
- assertEquals(7000, this.getOobCheckMillis(), "OOB check interval value is incorrect");
- assertEquals(4, this.getMaxMigrations(), "Max migrations is incorrect");
- assertEquals(10, this.getMaxOutstandingMigrations(), "Max outstanding migrations is incorrect");
- assertFalse(isIpBasedRegex());
- Map<String,Pattern> patterns = this.getPoolNameToRegexPattern();
- assertEquals(2, patterns.size());
- assertTrue(patterns.containsKey(FOO.getTableName()));
- assertEquals(Pattern.compile("r01.*").pattern(), patterns.get(FOO.getTableName()).pattern());
- assertTrue(patterns.containsKey(BAR.getTableName()));
- assertEquals(Pattern.compile("r02.*").pattern(), patterns.get(BAR.getTableName()).pattern());
- }
-
- @Test
- public void testBalance() {
- init();
- Set<KeyExtent> migrations = new HashSet<>();
- List<TabletMigration> migrationsOut = new ArrayList<>();
- long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
- migrationsOut);
- assertEquals(20000, wait);
- // should balance four tablets in one of the tables before reaching max
- assertEquals(4, migrationsOut.size());
-
- // now balance again passing in the new migrations
- for (TabletMigration m : migrationsOut) {
- migrations.add(m.tablet);
- }
- migrationsOut.clear();
- wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
- migrationsOut);
- assertEquals(20000, wait);
- // should balance four tablets in one of the other tables before reaching max
- assertEquals(4, migrationsOut.size());
-
- // now balance again passing in the new migrations
- for (TabletMigration m : migrationsOut) {
- migrations.add(m.tablet);
- }
- migrationsOut.clear();
- wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
- migrationsOut);
- assertEquals(20000, wait);
- // should balance four tablets in one of the other tables before reaching max
- assertEquals(4, migrationsOut.size());
-
- // now balance again passing in the new migrations
- for (TabletMigration m : migrationsOut) {
- migrations.add(m.tablet);
- }
- migrationsOut.clear();
- wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
- migrationsOut);
- assertEquals(20000, wait);
- // no more balancing to do
- assertEquals(0, migrationsOut.size());
- }
-
- @Test
- public void testBalanceWithTooManyOutstandingMigrations() {
- List<TabletMigration> migrationsOut = new ArrayList<>();
- init();
- // lets say we already have migrations ongoing for the FOO and BAR table extends (should be 5 of
- // each of them) for a total of 10
- Set<KeyExtent> migrations = new HashSet<>();
- migrations.addAll(tableExtents.get(FOO.getTableName()));
- migrations.addAll(tableExtents.get(BAR.getTableName()));
- long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
- migrationsOut);
- assertEquals(20000, wait);
- // no migrations should have occurred as 10 is the maxOutstandingMigrations
- assertEquals(0, migrationsOut.size());
- }
-
- @Test
- public void testSplitCurrentByRegexUsingHostname() {
- init();
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> groups =
- this.splitCurrentByRegex(createCurrent(15));
- assertEquals(3, groups.size());
- assertTrue(groups.containsKey(FOO.getTableName()));
- SortedMap<TServerInstance,TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
- assertEquals(5, fooHosts.size());
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
- assertTrue(groups.containsKey(BAR.getTableName()));
- SortedMap<TServerInstance,TabletServerStatus> barHosts = groups.get(BAR.getTableName());
- assertEquals(5, barHosts.size());
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
- assertTrue(groups.containsKey(DEFAULT_POOL));
- SortedMap<TServerInstance,TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
- assertEquals(5, defHosts.size());
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
- }
-
- @Test
- public void testSplitCurrentByRegexUsingOverlappingPools() {
- ServerContext context = createMockContext();
- replay(context);
- initFactory(new TestServerConfigurationFactory(context) {
- @Override
- public synchronized AccumuloConfiguration getSystemConfiguration() {
- HashMap<String,String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r.*");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(),
- "r01.*|r02.*");
- return new ConfigurationCopy(props);
- }
- });
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> groups =
- this.splitCurrentByRegex(createCurrent(15));
-
- // Groups foo, bar, and the default pool which contains all known hosts
- assertEquals(3, groups.size());
- assertTrue(groups.containsKey(FOO.getTableName()));
- assertTrue(groups.containsKey(DEFAULT_POOL));
- for (String pool : new String[] {FOO.getTableName(), DEFAULT_POOL}) {
- SortedMap<TServerInstance,TabletServerStatus> fooHosts = groups.get(pool);
- assertEquals(15, fooHosts.size());
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
- }
-
- assertTrue(groups.containsKey(BAR.getTableName()));
- SortedMap<TServerInstance,TabletServerStatus> barHosts = groups.get(BAR.getTableName());
- assertEquals(10, barHosts.size());
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
- }
-
- @Test
- public void testSplitCurrentByRegexUsingIP() {
- ServerContext context = createMockContext();
- replay(context);
- initFactory(new TestServerConfigurationFactory(context) {
- @Override
- public synchronized AccumuloConfiguration getSystemConfiguration() {
- HashMap<String,String> props = new HashMap<>();
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "30s");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS_KEY, "true");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(),
- "192\\.168\\.0\\.[1-5]");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(),
- "192\\.168\\.0\\.[6-9]|192\\.168\\.0\\.10");
- return new ConfigurationCopy(props);
- }
- });
- assertTrue(isIpBasedRegex());
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> groups =
- this.splitCurrentByRegex(createCurrent(15));
- assertEquals(3, groups.size());
- assertTrue(groups.containsKey(FOO.getTableName()));
- SortedMap<TServerInstance,TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
- assertEquals(5, fooHosts.size());
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
- assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
- assertTrue(groups.containsKey(BAR.getTableName()));
- SortedMap<TServerInstance,TabletServerStatus> barHosts = groups.get(BAR.getTableName());
- assertEquals(5, barHosts.size());
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
- assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
- assertTrue(groups.containsKey(DEFAULT_POOL));
- SortedMap<TServerInstance,TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
- assertEquals(5, defHosts.size());
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
- assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
- }
-
- @Test
- public void testAllUnassigned() {
- init();
- Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- for (List<KeyExtent> extents : tableExtents.values()) {
- for (KeyExtent ke : extents) {
- unassigned.put(ke, null);
- }
- }
- this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(15, assignments.size());
- // Ensure unique tservers
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
- if (e.getKey().equals(e2.getKey())) {
- continue;
- }
- if (e.getValue().equals(e2.getValue())) {
- fail("Assignment failure");
- }
- }
- }
- // Ensure assignments are correct
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- if (!tabletInBounds(e.getKey(), e.getValue())) {
- fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
- }
- }
- }
-
- @Test
- public void testAllAssigned() {
- init();
- Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(0, assignments.size());
- }
-
- @Test
- public void testPartiallyAssigned() {
- init();
- Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- int i = 0;
- for (List<KeyExtent> extents : tableExtents.values()) {
- for (KeyExtent ke : extents) {
- if ((i % 2) == 0) {
- unassigned.put(ke, null);
- }
- i++;
- }
- }
- this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(unassigned.size(), assignments.size());
- // Ensure unique tservers
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
- if (e.getKey().equals(e2.getKey())) {
- continue;
- }
- if (e.getValue().equals(e2.getValue())) {
- fail("Assignment failure");
- }
- }
- }
- // Ensure assignments are correct
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- if (!tabletInBounds(e.getKey(), e.getValue())) {
- fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
- }
- }
- }
-
- @Test
- public void testUnassignedWithNoTServers() {
- init();
- Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
- unassigned.put(ke, null);
- }
- SortedMap<TServerInstance,TabletServerStatus> current = createCurrent(15);
- // Remove the BAR tablet servers from current
- List<TServerInstance> removals = new ArrayList<>();
- for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- if (e.getKey().getHost().equals("192.168.0.6") || e.getKey().getHost().equals("192.168.0.7")
- || e.getKey().getHost().equals("192.168.0.8")
- || e.getKey().getHost().equals("192.168.0.9")
- || e.getKey().getHost().equals("192.168.0.10")) {
- removals.add(e.getKey());
- }
- }
- for (TServerInstance r : removals) {
- current.remove(r);
- }
- this.getAssignments(Collections.unmodifiableSortedMap(current),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(unassigned.size(), assignments.size());
- // Ensure assignments are correct
- // Ensure tablets are assigned in default pool
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- if (tabletInBounds(e.getKey(), e.getValue())) {
- fail("tablet unexpectedly in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
- }
- }
- }
-
- @Test
- public void testUnassignedWithNoDefaultPool() {
- init();
- Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
- Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
- unassigned.put(ke, null);
- }
-
- SortedMap<TServerInstance,TabletServerStatus> current = createCurrent(15);
- // Remove the BAR tablet servers and default pool from current
- List<TServerInstance> removals = new ArrayList<>();
- for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- if (e.getKey().getHost().equals("192.168.0.6") || e.getKey().getHost().equals("192.168.0.7")
- || e.getKey().getHost().equals("192.168.0.8")
- || e.getKey().getHost().equals("192.168.0.9")
- || e.getKey().getHost().equals("192.168.0.10")
- || e.getKey().getHost().equals("192.168.0.11")
- || e.getKey().getHost().equals("192.168.0.12")
- || e.getKey().getHost().equals("192.168.0.13")
- || e.getKey().getHost().equals("192.168.0.14")
- || e.getKey().getHost().equals("192.168.0.15")) {
- removals.add(e.getKey());
- }
- }
-
- for (TServerInstance r : removals) {
- current.remove(r);
- }
-
- this.getAssignments(Collections.unmodifiableSortedMap(current),
- Collections.unmodifiableMap(unassigned), assignments);
- assertEquals(unassigned.size(), assignments.size());
-
- // Ensure tablets are assigned in default pool
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- if (tabletInBounds(e.getKey(), e.getValue())) {
- fail("tablet unexpectedly in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
- }
- }
- }
-
- @Test
- public void testOutOfBoundsTablets() {
- init();
- // Wait to trigger the out of bounds check which will call our version of
- // getOnlineTabletsForTable
- UtilWaitThread.sleep(11000);
- Set<KeyExtent> migrations = new HashSet<>();
- List<TabletMigration> migrationsOut = new ArrayList<>();
- this.balance(createCurrent(15), migrations, migrationsOut);
- assertEquals(2, migrationsOut.size());
- }
-
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId) {
- // Report incorrect information so that balance will create an assignment
- List<TabletStats> tablets = new ArrayList<>();
- if (tableId.equals(BAR.getId()) && tserver.getHost().equals("192.168.0.1")) {
- // Report that we have a bar tablet on this server
- TKeyExtent tke = new TKeyExtent();
- tke.setTable(BAR.getId().canonical().getBytes(UTF_8));
- tke.setEndRow("11".getBytes());
- tke.setPrevEndRow("10".getBytes());
- TabletStats ts = new TabletStats();
- ts.setExtent(tke);
- tablets.add(ts);
- } else if (tableId.equals(FOO.getId()) && tserver.getHost().equals("192.168.0.6")) {
- // Report that we have a foo tablet on this server
- TKeyExtent tke = new TKeyExtent();
- tke.setTable(FOO.getId().canonical().getBytes(UTF_8));
- tke.setEndRow("1".getBytes());
- tke.setPrevEndRow("0".getBytes());
- TabletStats ts = new TabletStats();
- ts.setExtent(tke);
- tablets.add(ts);
- }
- return tablets;
- }
-
-}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java
deleted file mode 100644
index de9718baa0..0000000000
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.master.balancer;
-
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.InstanceId;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.server.MockServerContext;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.hadoop.io.Text;
-import org.easymock.EasyMock;
-import org.junit.jupiter.api.Test;
-
-@Deprecated(since = "2.1.0")
-public class TableLoadBalancerTest {
-
- private static Map<String,String> TABLE_ID_MAP = Map.of("t1", "a1", "t2", "b12", "t3", "c4");
-
- private static TServerInstance mkts(String address, String session) {
- return new TServerInstance(HostAndPort.fromParts(address, 1234), session);
- }
-
- private static TabletServerStatus status(Object... config) {
- TabletServerStatus result = new TabletServerStatus();
- result.tableMap = new HashMap<>();
- String tablename = null;
- for (Object c : config) {
- if (c instanceof String) {
- tablename = (String) c;
- } else {
- TableInfo info = new TableInfo();
- int count = (Integer) c;
- info.onlineTablets = count;
- info.tablets = count;
- result.tableMap.put(tablename, info);
- }
- }
- return result;
- }
-
- static SortedMap<TServerInstance,TabletServerStatus> state;
-
- static List<TabletStats> generateFakeTablets(TServerInstance tserver, TableId tableId) {
- List<TabletStats> result = new ArrayList<>();
- TabletServerStatus tableInfo = state.get(tserver);
- // generate some fake tablets
- for (int i = 0; i < tableInfo.tableMap.get(tableId.canonical()).onlineTablets; i++) {
- TabletStats stats = new TabletStats();
- stats.extent =
- new KeyExtent(tableId, new Text(tserver.getHost() + String.format("%03d", i + 1)),
- new Text(tserver.getHost() + String.format("%03d", i))).toThrift();
- result.add(stats);
- }
- return result;
- }
-
- static class DefaultLoadBalancer
- extends org.apache.accumulo.server.master.balancer.DefaultLoadBalancer {
-
- public DefaultLoadBalancer(TableId table) {
- super(table);
- }
-
- @Override
- public void init(ServerContext context) {}
-
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId) {
- return generateFakeTablets(tserver, tableId);
- }
- }
-
- // ugh... so wish I had provided mock objects to the LoadBalancer in the manager
- class TableLoadBalancer extends org.apache.accumulo.server.master.balancer.TableLoadBalancer {
-
- // use our new classname to test class loading
- @Override
- protected String getLoadBalancerClassNameForTable(TableId table) {
- return DefaultLoadBalancer.class.getName();
- }
-
- // we don't have real tablet servers to ask: invent some online tablets
- @Override
- public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, TableId tableId) {
- return generateFakeTablets(tserver, tableId);
- }
-
- @Override
- protected TableOperations getTableOperations() {
- TableOperations tops = createMock(TableOperations.class);
- expect(tops.tableIdMap()).andReturn(TABLE_ID_MAP).anyTimes();
- replay(tops);
- return tops;
- }
- }
-
- private ServerContext createMockContext() {
- final InstanceId instanceId =
- InstanceId.of(UUID.nameUUIDFromBytes(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
- return MockServerContext.getWithZK(instanceId, "10.0.0.1:1234", 30_000);
- }
-
- @Test
- public void test() {
- final ServerContext context = createMockContext();
- TableConfiguration conf = createMock(TableConfiguration.class);
- // Eclipse might show @SuppressWarnings("removal") as unnecessary.
- // Eclipse is wrong. See https://bugs.eclipse.org/bugs/show_bug.cgi?id=565271
- @SuppressWarnings("removal")
- Property TABLE_CLASSPATH = Property.TABLE_CLASSPATH;
- expect(conf.resolve(Property.TABLE_CLASSLOADER_CONTEXT, TABLE_CLASSPATH))
- .andReturn(Property.TABLE_CLASSLOADER_CONTEXT).anyTimes();
- expect(conf.get(Property.TABLE_CLASSLOADER_CONTEXT)).andReturn("").anyTimes();
- expect(context.getTableConfiguration(EasyMock.anyObject())).andReturn(conf).anyTimes();
- replay(context, conf);
-
- String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"),
- t3Id = TABLE_ID_MAP.get("t3");
- state = new TreeMap<>();
- TServerInstance svr = mkts("10.0.0.1", "0x01020304");
- state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
-
- Set<KeyExtent> migrations = Collections.emptySet();
- List<TabletMigration> migrationsOut = new ArrayList<>();
- TableLoadBalancer tls = new TableLoadBalancer();
- tls.init(context);
- tls.balance(state, migrations, migrationsOut);
- assertEquals(0, migrationsOut.size());
-
- state.put(mkts("10.0.0.2", "0x02030405"), status());
- tls = new TableLoadBalancer();
- tls.init(context);
- tls.balance(state, migrations, migrationsOut);
- int count = 0;
- Map<TableId,Integer> movedByTable = new HashMap<>();
- movedByTable.put(TableId.of(t1Id), 0);
- movedByTable.put(TableId.of(t2Id), 0);
- movedByTable.put(TableId.of(t3Id), 0);
- for (TabletMigration migration : migrationsOut) {
- if (migration.oldServer.equals(svr)) {
- count++;
- }
- TableId key = migration.tablet.tableId();
- movedByTable.put(key, movedByTable.get(key) + 1);
- }
- assertEquals(15, count);
- for (Integer moved : movedByTable.values()) {
- assertEquals(5, moved.intValue());
- }
- }
-
-}