You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/04/11 19:56:58 UTC
svn commit: r1467010 - in /accumulo/branches/1.5:
server/src/main/java/org/apache/accumulo/server/master/
server/src/main/java/org/apache/accumulo/server/master/state/
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/
test/src/test/jav...
Author: ecn
Date: Thu Apr 11 17:56:57 2013
New Revision: 1467010
URL: http://svn.apache.org/r1467010
Log:
ACCUMULO-1264 tweak the scan range when looking for !METADATA tablet consistency check; add !METADATA table merge/split to the randomwalk test
Added:
accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java (with props)
Modified:
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Apr 11 17:56:57 2013
@@ -1203,6 +1203,7 @@ public class Master implements LiveTServ
}
// Handle merge transitions
if (mergeInfo.getRange() != null) {
+ log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
if (mergeInfo.overlaps(extent)) {
switch (mergeInfo.getState()) {
case NONE:
@@ -2094,8 +2095,11 @@ public class Master implements LiveTServ
});
TCredentials systemAuths = SecurityConstants.getSystemCredentials();
- final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new RootTabletStateStore(instance, systemAuths, this),
- new MetaDataStateStore(instance, systemAuths, this)};
+ final TabletStateStore stores[] = {
+ new ZooTabletStateStore(new ZooStore(zroot)),
+ new RootTabletStateStore(instance, systemAuths, this),
+ new MetaDataStateStore(instance, systemAuths, this)
+ };
watchers.add(new TabletGroupWatcher(stores[2], null));
watchers.add(new TabletGroupWatcher(stores[1], watchers.get(0)));
watchers.add(new TabletGroupWatcher(stores[0], watchers.get(1)));
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java Thu Apr 11 17:56:57 2013
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.client.S
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -182,11 +183,14 @@ public class MergeStats {
Text tableId = extent.getTableId();
Text first = KeyExtent.getMetadataEntry(tableId, start);
Range range = new Range(first, false, null, true);
- if (extent.isMeta())
- range = new Range(Constants.METADATA_ROOT_TABLET_KEYSPACE);
+ if (extent.isMeta()) {
+ // don't go off the root tablet
+ range = new Range(new Key(first).followingKey(PartialKey.ROW), false, Constants.METADATA_ROOT_TABLET_KEYSPACE.getEndKey(), false);
+ }
scanner.setRange(range);
KeyExtent prevExtent = null;
+ log.debug("Scanning range " + range);
for (Entry<Key,Value> entry : scanner) {
TabletLocationState tls;
try {
@@ -195,24 +199,30 @@ public class MergeStats {
log.error(e, e);
return false;
}
+ log.debug("consistency check: " + tls + " walogs " + tls.walogs.size());
if (!tls.extent.getTableId().equals(tableId)) {
break;
}
if (!tls.walogs.isEmpty() && verify.getMergeInfo().needsToBeChopped(tls.extent)) {
+ log.debug("failing consistency: needs to be chopped" + tls.extent);
return false;
}
if (prevExtent == null) {
// this is the first tablet observed, it must be offline and its prev row must be less than the start of the merge range
if (tls.extent.getPrevEndRow() != null && tls.extent.getPrevEndRow().compareTo(start) > 0) {
+ log.debug("failing consistency: prev row is too high " + start);
return false;
}
- if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED)
+ if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED) {
+ log.debug("failing consistency: assigned or hosted " + tls);
return false;
+ }
} else if (!tls.extent.isPreviousExtent(prevExtent)) {
+ log.debug("hole in !METADATA");
return false;
}
@@ -224,6 +234,9 @@ public class MergeStats {
break;
}
}
+ log.debug("chopped " + chopped + " v.chopped " + verify.chopped +
+ " unassigned " + unassigned + " v.unassigned " + verify.unassigned +
+ " verify.total " + verify.total);
return chopped == verify.chopped && unassigned == verify.unassigned && unassigned == verify.total;
}
Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java (original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java Thu Apr 11 17:56:57 2013
@@ -16,11 +16,13 @@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.TableOfflineException;
@@ -38,7 +40,8 @@ public class AddSplits extends Test {
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
-
+ tableNames = new ArrayList<String>(tableNames);
+ tableNames.add(Constants.METADATA_TABLE_NAME);
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
TreeSet<Text> splits = new TreeSet<Text>();
Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java (original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java Thu Apr 11 17:56:57 2013
@@ -16,11 +16,13 @@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.TableOfflineException;
@@ -38,7 +40,8 @@ public class Merge extends Test {
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
-
+ tableNames = new ArrayList<String>(tableNames);
+ tableNames.add(Constants.METADATA_TABLE_NAME);
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
// TODO need to sometimes do null start and end ranges
Added: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java?rev=1467010&view=auto
==============================================================================
--- accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java (added)
+++ accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java Thu Apr 11 17:56:57 2013
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.io.Text;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MetaSplitTest {
+
+ private static String secret = "superSecret";
+ public static TemporaryFolder folder = new TemporaryFolder();
+ public static MiniAccumuloCluster cluster;
+
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ folder.create();
+ MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
+ cluster = new MiniAccumuloCluster(cfg);
+ cluster.start();
+
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cluster.stop();
+ folder.delete();
+ }
+
+ private void addSplits(TableOperations opts, String ... points) throws Exception {
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (String point : points) {
+ splits.add(new Text(point));
+ }
+ opts.addSplits(Constants.METADATA_TABLE_NAME, splits);
+ }
+
+ @Test(timeout = 60000)
+ public void testMetaSplit() throws Exception {
+ Instance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
+ Connector connector = instance.getConnector("root", new PasswordToken(secret));
+ TableOperations opts = connector.tableOperations();
+ for (int i = 1; i <= 10; i++) {
+ opts.create("" + i);
+ }
+ assertEquals(2, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ addSplits(opts, "4 5 6 7 8".split(" "));
+ assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ opts.merge(Constants.METADATA_TABLE_NAME, new Text("6"), new Text("9"));
+ assertEquals(5, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ addSplits(opts, "44 55 66 77 88".split(" "));
+ assertEquals(10, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ opts.merge(Constants.METADATA_TABLE_NAME, new Text("5"), new Text("7"));
+ assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ opts.merge(Constants.METADATA_TABLE_NAME, null, null);
+ assertEquals(1, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+ }
+
+}
Propchange: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java
------------------------------------------------------------------------------
svn:eol-style = native