You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/07/17 20:22:07 UTC
svn commit: r1504222 - in /hbase/branches/0.89-fb: ./
src/main/java/org/apache/hadoop/hbase/master/
src/test/java/org/apache/hadoop/hbase/master/
Author: liyin
Date: Wed Jul 17 18:22:07 2013
New Revision: 1504222
URL: http://svn.apache.org/r1504222
Log:
[master]Add tool to update assignment plan stored in json file
Author: fan
Summary: Add tool to update assignment plan stored in json file
Test Plan:
Added a unit test to reconstruct an assignment plan from json string of current plan.
I tested download and upload functions in shadow cluster 52 through following steps:
1. Download current assignment plan.
2. Modify the plan and upload it.
3. Download new plan and diff with original one, only changes from last step should exist.
4. Upload original plan.
3. Download current plan and diff with original one, they should be the same.
The test steps have been passed.
Reviewers: liyintang, adela
Reviewed By: adela
CC: security-diffs@lists, hbase-eng@
Differential Revision: https://phabricator.fb.com/D879881
Task ID: 2547614
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/AssignmentPlanData.java
Modified:
hbase/branches/0.89-fb/pom.xml
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
Modified: hbase/branches/0.89-fb/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/pom.xml?rev=1504222&r1=1504221&r2=1504222&view=diff
==============================================================================
--- hbase/branches/0.89-fb/pom.xml (original)
+++ hbase/branches/0.89-fb/pom.xml Wed Jul 17 18:22:07 2013
@@ -780,6 +780,12 @@
<version>${jruby.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.2.4</version>
+ </dependency>
+
<!-- REST dependencies -->
<dependency>
<groupId>com.google.protobuf</groupId>
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/AssignmentPlanData.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/AssignmentPlanData.java?rev=1504222&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/AssignmentPlanData.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/AssignmentPlanData.java Wed Jul 17 18:22:07 2013
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2013 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Data object that helps map assignment plans to json format.
+ * Example json file:
+ * {
+ * "assignments" : [
+ * {"regionname" : "regionX", "favored" : ["host1:port1", "host2:port2", "host3:port3"]},
+ * {"regionname" : "regionY", "favored" : ["host4:port4", "host5:port5", "host6:port6"]}
+ * ]
+ * }
+ */
+public class AssignmentPlanData {
+ protected static final Log LOG = LogFactory.getLog(AssignmentPlanData.class.getName());
+
+ private List<Assignment> assignments;
+
+ public AssignmentPlanData(List<Assignment> assignments) {
+ this.assignments = assignments;
+ }
+
+ public List<Assignment> getAssignments() {
+ return assignments;
+ }
+
+ /**
+ * Contains essential parameters for a region placement.
+ */
+ public static class Assignment {
+ private String regionname;
+ private List<String> favored;
+
+ public Assignment(String regionname, List<String> favored) {
+ this.regionname = regionname;
+ this.favored = favored;
+ }
+
+ public String getRegionname() {
+ return regionname;
+ }
+
+ public List<String> getFavored() {
+ return favored;
+ }
+ }
+
+ /**
+ * Convert assignment plan into data object which is used in json serialization
+ * @param plan
+ * @return assignment plan for each region as list, wrapped in data object
+ */
+ public static AssignmentPlanData constructFromAssignmentPlan(AssignmentPlan plan) {
+ List<Assignment> assignments = new ArrayList<Assignment>();
+
+ Map<HRegionInfo, List<HServerAddress>> lists = plan.getAssignmentMap();
+ for (Map.Entry<HRegionInfo, List<HServerAddress>> entry : lists.entrySet()) {
+ String regionname = entry.getKey().getRegionNameAsString();
+ List<String> addresses = new ArrayList<String>();
+ for (HServerAddress address : entry.getValue()) {
+ addresses.add(address.getHostNameWithPort());
+ }
+ Assignment assignment = new Assignment(regionname, addresses);
+ assignments.add(assignment);
+ }
+
+ return new AssignmentPlanData(assignments);
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java?rev=1504222&r1=1504221&r2=1504222&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionPlacement.java Wed Jul 17 18:22:07 2013
@@ -1,24 +1,33 @@
+/**
+ * Copyright 2013 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.master;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.TreeMap;
-
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonSyntaxException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,6 +50,21 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.TreeMap;
+
public class RegionPlacement implements RegionPlacementPolicy{
private static final Log LOG = LogFactory.getLog(RegionPlacement.class
.getName());
@@ -1275,6 +1299,11 @@ public class RegionPlacement implements
opt.addOption("r", true, "The region name that needs to be updated");
opt.addOption("f", true, "The new favored nodes");
+ opt.addOption("upload", true,
+ "upload the json file which contains new placement plan");
+ opt.addOption("download", true,
+ "download existing placement plan into a json file");
+
opt.addOption("tables", true,
"The list of table names splitted by ',' ;" +
"For example: -tables: t1,t2,...,tn");
@@ -1433,6 +1462,41 @@ public class RegionPlacement implements
String newRack = s.nextLine().trim();
s.close();
rp.expandRegionsToNewRack(newRack, snapshot);
+ } else if (cmd.hasOption("upload")) {
+ String fileName = cmd.getOptionValue("upload");
+ try {
+ String jsonStr = FileUtils.readFileToString(new File(fileName));
+ AssignmentPlan newPlan = rp.loadPlansFromJson(jsonStr);
+ Map<String, Map<String, Float>> locality = FSUtils
+ .getRegionDegreeLocalityMappingFromFS(conf);
+ Map<String, Integer> movesPerTable = rp.getRegionsMovement(newPlan);
+ rp.checkDifferencesWithOldPlan(movesPerTable, locality, newPlan);
+ System.out.println("Do you want to update the assignment plan? [y/n]");
+ Scanner s = new Scanner(System.in);
+ String input = s.nextLine().trim();
+ if (input.equals("y")) {
+ System.out.println("Updating assignment plan...");
+ rp.updateAssignmentPlan(newPlan);
+ }
+ s.close();
+ } catch (IOException e) {
+ LOG.error("Unable to load plan file: " + e);
+ } catch (JsonSyntaxException je) {
+ LOG.error("Unable to parse json file: " + je);
+ }
+ } else if (cmd.hasOption("download")) {
+ String path = cmd.getOptionValue("download");
+ try {
+ RegionAssignmentSnapshot snapshot = rp.getRegionAssignmentSnapshot();
+ AssignmentPlan plan = snapshot.getExistingAssignmentPlan();
+ AssignmentPlanData data = AssignmentPlanData.constructFromAssignmentPlan(plan);
+ Gson prettyGson = new GsonBuilder().setPrettyPrinting().create();
+ String jsonOutput = prettyGson.toJson(data);
+ FileUtils.write(new File(path), jsonOutput);
+ } catch (Exception e) {
+ LOG.error("Unable to download current assignment plan" + e);
+ e.printStackTrace();
+ }
} else {
printHelp(opt);
}
@@ -1761,4 +1825,37 @@ public class RegionPlacement implements
printDispersionScores(table, snapshot, regions.size(), null, false);
}
}
+
+ /**
+ * Convert json string to assignment plan
+ * @param jsonStr
+ * @return assignment plan converted from json string
+ * @throws JsonSyntaxException
+ * @throws IOException
+ */
+ public AssignmentPlan loadPlansFromJson(String jsonStr)
+ throws JsonSyntaxException, IOException {
+ AssignmentPlanData data = new Gson().fromJson(jsonStr, AssignmentPlanData.class);
+ AssignmentPlan newPlan = new AssignmentPlan();
+ RegionAssignmentSnapshot snapshot = this.getRegionAssignmentSnapshot();
+ Map<String, HRegionInfo> map = snapshot.getRegionNameToRegionInfoMap();
+ for (AssignmentPlanData.Assignment plan : data.getAssignments()) {
+ HRegionInfo regionInfo = map.get(plan.getRegionname());
+ List<HServerAddress> favoredNodes = null;
+ if (regionInfo == null) {
+ LOG.error("Cannot find the region " + plan.getRegionname());
+ } else {
+ List<String> addresses = plan.getFavored();
+ String nodesStr = StringUtils.join(addresses, ",");
+ try {
+ favoredNodes = RegionPlacement.getFavoredNodeList(nodesStr);
+ } catch (IllegalArgumentException e) {
+ LOG.error("Cannot parse the invalid favored nodes because " + e);
+ }
+ }
+ newPlan.updateAssignmentPlan(regionInfo, favoredNodes);
+ }
+
+ return newPlan;
+ }
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java?rev=1504222&r1=1504221&r2=1504222&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java Wed Jul 17 18:22:07 2013
@@ -19,21 +19,7 @@
*/
package org.apache.hadoop.hbase.master;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.gson.Gson;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -60,6 +46,21 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
public class TestRegionPlacement {
final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -552,4 +553,28 @@ public class TestRegionPlacement {
throw new RuntimeException();
}
}
+
+ /**
+ * Download current assignment plan, serialize it to json and deserialize the json.
+ * The two plans should be identical.
+ */
+ @Test
+ public void testJsonToAP() {
+ try {
+ createTable("testJsonAssignmentPlan", 3);
+
+ AssignmentPlan currentPlan = rp.getExistingAssignmentPlan();
+ RegionPlacement.printAssignmentPlan(currentPlan);
+ AssignmentPlanData data = AssignmentPlanData.constructFromAssignmentPlan(currentPlan);
+
+ String jsonStr = new Gson().toJson(data);
+ LOG.info("Json version of current assignment plan: " + jsonStr);
+ AssignmentPlan loadedPlan = rp.loadPlansFromJson(jsonStr);
+ RegionPlacement.printAssignmentPlan(loadedPlan);
+ assertEquals("Loaded plan should be the same with current plan", currentPlan, loadedPlan);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ }
}